1 //===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This implements the SelectionDAG class. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/CodeGen/SelectionDAG.h" 15 #include "SDNodeDbgValue.h" 16 #include "llvm/ADT/APFloat.h" 17 #include "llvm/ADT/APInt.h" 18 #include "llvm/ADT/APSInt.h" 19 #include "llvm/ADT/ArrayRef.h" 20 #include "llvm/ADT/BitVector.h" 21 #include "llvm/ADT/FoldingSet.h" 22 #include "llvm/ADT/None.h" 23 #include "llvm/ADT/STLExtras.h" 24 #include "llvm/ADT/SmallPtrSet.h" 25 #include "llvm/ADT/SmallVector.h" 26 #include "llvm/ADT/Triple.h" 27 #include "llvm/ADT/Twine.h" 28 #include "llvm/Analysis/ValueTracking.h" 29 #include "llvm/CodeGen/ISDOpcodes.h" 30 #include "llvm/CodeGen/MachineBasicBlock.h" 31 #include "llvm/CodeGen/MachineConstantPool.h" 32 #include "llvm/CodeGen/MachineFrameInfo.h" 33 #include "llvm/CodeGen/MachineFunction.h" 34 #include "llvm/CodeGen/MachineMemOperand.h" 35 #include "llvm/CodeGen/RuntimeLibcalls.h" 36 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h" 37 #include "llvm/CodeGen/SelectionDAGNodes.h" 38 #include "llvm/CodeGen/SelectionDAGTargetInfo.h" 39 #include "llvm/CodeGen/TargetLowering.h" 40 #include "llvm/CodeGen/TargetRegisterInfo.h" 41 #include "llvm/CodeGen/TargetSubtargetInfo.h" 42 #include "llvm/CodeGen/ValueTypes.h" 43 #include "llvm/IR/Constant.h" 44 #include "llvm/IR/Constants.h" 45 #include "llvm/IR/DataLayout.h" 46 #include "llvm/IR/DebugInfoMetadata.h" 47 #include "llvm/IR/DebugLoc.h" 48 #include "llvm/IR/DerivedTypes.h" 49 #include "llvm/IR/Function.h" 50 #include "llvm/IR/GlobalValue.h" 51 #include "llvm/IR/Metadata.h" 52 #include "llvm/IR/Type.h" 53 #include "llvm/IR/Value.h" 54 #include "llvm/Support/Casting.h" 55 #include "llvm/Support/CodeGen.h" 56 #include "llvm/Support/Compiler.h" 57 #include "llvm/Support/Debug.h" 58 #include "llvm/Support/ErrorHandling.h" 59 #include "llvm/Support/KnownBits.h" 60 #include "llvm/Support/MachineValueType.h" 61 #include "llvm/Support/ManagedStatic.h" 62 #include "llvm/Support/MathExtras.h" 63 #include "llvm/Support/Mutex.h" 64 #include "llvm/Support/raw_ostream.h" 65 #include "llvm/Target/TargetMachine.h" 66 #include "llvm/Target/TargetOptions.h" 67 #include <algorithm> 68 #include <cassert> 69 #include <cstdint> 70 #include <cstdlib> 71 #include <limits> 72 #include <set> 73 #include <string> 74 #include <utility> 75 #include <vector> 76 77 using namespace llvm; 78 79 /// makeVTList - Return an instance of the SDVTList struct initialized with the 80 /// specified members. 81 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { 82 SDVTList Res = {VTs, NumVTs}; 83 return Res; 84 } 85 86 // Default null implementations of the callbacks. 87 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {} 88 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {} 89 90 #define DEBUG_TYPE "selectiondag" 91 92 static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt", 93 cl::Hidden, cl::init(true), 94 cl::desc("Gang up loads and stores generated by inlining of memcpy")); 95 96 static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max", 97 cl::desc("Number limit for gluing ld/st of memcpy."), 98 cl::Hidden, cl::init(0)); 99 100 static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) { 101 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G);); 102 } 103 104 //===----------------------------------------------------------------------===// 105 // ConstantFPSDNode Class 106 //===----------------------------------------------------------------------===// 107 108 /// isExactlyValue - We don't rely on operator== working on double values, as 109 /// it returns true for things that are clearly not equal, like -0.0 and 0.0. 110 /// As such, this method can be used to do an exact bit-for-bit comparison of 111 /// two floating point values. 112 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { 113 return getValueAPF().bitwiseIsEqual(V); 114 } 115 116 bool ConstantFPSDNode::isValueValidForType(EVT VT, 117 const APFloat& Val) { 118 assert(VT.isFloatingPoint() && "Can only convert between FP types"); 119 120 // convert modifies in place, so make a copy. 121 APFloat Val2 = APFloat(Val); 122 bool losesInfo; 123 (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT), 124 APFloat::rmNearestTiesToEven, 125 &losesInfo); 126 return !losesInfo; 127 } 128 129 //===----------------------------------------------------------------------===// 130 // ISD Namespace 131 //===----------------------------------------------------------------------===// 132 133 bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) { 134 auto *BV = dyn_cast<BuildVectorSDNode>(N); 135 if (!BV) 136 return false; 137 138 APInt SplatUndef; 139 unsigned SplatBitSize; 140 bool HasUndefs; 141 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits(); 142 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs, 143 EltSize) && 144 EltSize == SplatBitSize; 145 } 146 147 // FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be 148 // specializations of the more general isConstantSplatVector()? 149 150 bool ISD::isBuildVectorAllOnes(const SDNode *N) { 151 // Look through a bit convert. 152 while (N->getOpcode() == ISD::BITCAST) 153 N = N->getOperand(0).getNode(); 154 155 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 156 157 unsigned i = 0, e = N->getNumOperands(); 158 159 // Skip over all of the undef values. 160 while (i != e && N->getOperand(i).isUndef()) 161 ++i; 162 163 // Do not accept an all-undef vector. 164 if (i == e) return false; 165 166 // Do not accept build_vectors that aren't all constants or which have non-~0 167 // elements. We have to be a bit careful here, as the type of the constant 168 // may not be the same as the type of the vector elements due to type 169 // legalization (the elements are promoted to a legal type for the target and 170 // a vector of a type may be legal when the base element type is not). 171 // We only want to check enough bits to cover the vector elements, because 172 // we care if the resultant vector is all ones, not whether the individual 173 // constants are. 174 SDValue NotZero = N->getOperand(i); 175 unsigned EltSize = N->getValueType(0).getScalarSizeInBits(); 176 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) { 177 if (CN->getAPIntValue().countTrailingOnes() < EltSize) 178 return false; 179 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) { 180 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize) 181 return false; 182 } else 183 return false; 184 185 // Okay, we have at least one ~0 value, check to see if the rest match or are 186 // undefs. Even with the above element type twiddling, this should be OK, as 187 // the same type legalization should have applied to all the elements. 188 for (++i; i != e; ++i) 189 if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef()) 190 return false; 191 return true; 192 } 193 194 bool ISD::isBuildVectorAllZeros(const SDNode *N) { 195 // Look through a bit convert. 196 while (N->getOpcode() == ISD::BITCAST) 197 N = N->getOperand(0).getNode(); 198 199 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 200 201 bool IsAllUndef = true; 202 for (const SDValue &Op : N->op_values()) { 203 if (Op.isUndef()) 204 continue; 205 IsAllUndef = false; 206 // Do not accept build_vectors that aren't all constants or which have non-0 207 // elements. We have to be a bit careful here, as the type of the constant 208 // may not be the same as the type of the vector elements due to type 209 // legalization (the elements are promoted to a legal type for the target 210 // and a vector of a type may be legal when the base element type is not). 211 // We only want to check enough bits to cover the vector elements, because 212 // we care if the resultant vector is all zeros, not whether the individual 213 // constants are. 214 unsigned EltSize = N->getValueType(0).getScalarSizeInBits(); 215 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) { 216 if (CN->getAPIntValue().countTrailingZeros() < EltSize) 217 return false; 218 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) { 219 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize) 220 return false; 221 } else 222 return false; 223 } 224 225 // Do not accept an all-undef vector. 226 if (IsAllUndef) 227 return false; 228 return true; 229 } 230 231 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) { 232 if (N->getOpcode() != ISD::BUILD_VECTOR) 233 return false; 234 235 for (const SDValue &Op : N->op_values()) { 236 if (Op.isUndef()) 237 continue; 238 if (!isa<ConstantSDNode>(Op)) 239 return false; 240 } 241 return true; 242 } 243 244 bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) { 245 if (N->getOpcode() != ISD::BUILD_VECTOR) 246 return false; 247 248 for (const SDValue &Op : N->op_values()) { 249 if (Op.isUndef()) 250 continue; 251 if (!isa<ConstantFPSDNode>(Op)) 252 return false; 253 } 254 return true; 255 } 256 257 bool ISD::allOperandsUndef(const SDNode *N) { 258 // Return false if the node has no operands. 259 // This is "logically inconsistent" with the definition of "all" but 260 // is probably the desired behavior. 261 if (N->getNumOperands() == 0) 262 return false; 263 264 for (const SDValue &Op : N->op_values()) 265 if (!Op.isUndef()) 266 return false; 267 268 return true; 269 } 270 271 bool ISD::matchUnaryPredicate(SDValue Op, 272 std::function<bool(ConstantSDNode *)> Match) { 273 if (auto *Cst = dyn_cast<ConstantSDNode>(Op)) 274 return Match(Cst); 275 276 if (ISD::BUILD_VECTOR != Op.getOpcode()) 277 return false; 278 279 EVT SVT = Op.getValueType().getScalarType(); 280 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) { 281 auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(i)); 282 if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst)) 283 return false; 284 } 285 return true; 286 } 287 288 bool ISD::matchBinaryPredicate( 289 SDValue LHS, SDValue RHS, 290 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match) { 291 if (LHS.getValueType() != RHS.getValueType()) 292 return false; 293 294 if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS)) 295 if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS)) 296 return Match(LHSCst, RHSCst); 297 298 if (ISD::BUILD_VECTOR != LHS.getOpcode() || 299 ISD::BUILD_VECTOR != RHS.getOpcode()) 300 return false; 301 302 EVT SVT = LHS.getValueType().getScalarType(); 303 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) { 304 auto *LHSCst = dyn_cast<ConstantSDNode>(LHS.getOperand(i)); 305 auto *RHSCst = dyn_cast<ConstantSDNode>(RHS.getOperand(i)); 306 if (!LHSCst || !RHSCst) 307 return false; 308 if (LHSCst->getValueType(0) != SVT || 309 LHSCst->getValueType(0) != RHSCst->getValueType(0)) 310 return false; 311 if (!Match(LHSCst, RHSCst)) 312 return false; 313 } 314 return true; 315 } 316 317 ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) { 318 switch (ExtType) { 319 case ISD::EXTLOAD: 320 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND; 321 case ISD::SEXTLOAD: 322 return ISD::SIGN_EXTEND; 323 case ISD::ZEXTLOAD: 324 return ISD::ZERO_EXTEND; 325 default: 326 break; 327 } 328 329 llvm_unreachable("Invalid LoadExtType"); 330 } 331 332 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { 333 // To perform this operation, we just need to swap the L and G bits of the 334 // operation. 335 unsigned OldL = (Operation >> 2) & 1; 336 unsigned OldG = (Operation >> 1) & 1; 337 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits 338 (OldL << 1) | // New G bit 339 (OldG << 2)); // New L bit. 340 } 341 342 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) { 343 unsigned Operation = Op; 344 if (isInteger) 345 Operation ^= 7; // Flip L, G, E bits, but not U. 346 else 347 Operation ^= 15; // Flip all of the condition bits. 348 349 if (Operation > ISD::SETTRUE2) 350 Operation &= ~8; // Don't let N and U bits get set. 351 352 return ISD::CondCode(Operation); 353 } 354 355 /// For an integer comparison, return 1 if the comparison is a signed operation 356 /// and 2 if the result is an unsigned comparison. Return zero if the operation 357 /// does not depend on the sign of the input (setne and seteq). 358 static int isSignedOp(ISD::CondCode Opcode) { 359 switch (Opcode) { 360 default: llvm_unreachable("Illegal integer setcc operation!"); 361 case ISD::SETEQ: 362 case ISD::SETNE: return 0; 363 case ISD::SETLT: 364 case ISD::SETLE: 365 case ISD::SETGT: 366 case ISD::SETGE: return 1; 367 case ISD::SETULT: 368 case ISD::SETULE: 369 case ISD::SETUGT: 370 case ISD::SETUGE: return 2; 371 } 372 } 373 374 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, 375 bool IsInteger) { 376 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 377 // Cannot fold a signed integer setcc with an unsigned integer setcc. 378 return ISD::SETCC_INVALID; 379 380 unsigned Op = Op1 | Op2; // Combine all of the condition bits. 381 382 // If the N and U bits get set, then the resultant comparison DOES suddenly 383 // care about orderedness, and it is true when ordered. 384 if (Op > ISD::SETTRUE2) 385 Op &= ~16; // Clear the U bit if the N bit is set. 386 387 // Canonicalize illegal integer setcc's. 388 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT 389 Op = ISD::SETNE; 390 391 return ISD::CondCode(Op); 392 } 393 394 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, 395 bool IsInteger) { 396 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 397 // Cannot fold a signed setcc with an unsigned setcc. 398 return ISD::SETCC_INVALID; 399 400 // Combine all of the condition bits. 401 ISD::CondCode Result = ISD::CondCode(Op1 & Op2); 402 403 // Canonicalize illegal integer setcc's. 404 if (IsInteger) { 405 switch (Result) { 406 default: break; 407 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT 408 case ISD::SETOEQ: // SETEQ & SETU[LG]E 409 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE 410 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE 411 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE 412 } 413 } 414 415 return Result; 416 } 417 418 //===----------------------------------------------------------------------===// 419 // SDNode Profile Support 420 //===----------------------------------------------------------------------===// 421 422 /// AddNodeIDOpcode - Add the node opcode to the NodeID data. 423 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { 424 ID.AddInteger(OpC); 425 } 426 427 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them 428 /// solely with their pointer. 429 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { 430 ID.AddPointer(VTList.VTs); 431 } 432 433 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 434 static void AddNodeIDOperands(FoldingSetNodeID &ID, 435 ArrayRef<SDValue> Ops) { 436 for (auto& Op : Ops) { 437 ID.AddPointer(Op.getNode()); 438 ID.AddInteger(Op.getResNo()); 439 } 440 } 441 442 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 443 static void AddNodeIDOperands(FoldingSetNodeID &ID, 444 ArrayRef<SDUse> Ops) { 445 for (auto& Op : Ops) { 446 ID.AddPointer(Op.getNode()); 447 ID.AddInteger(Op.getResNo()); 448 } 449 } 450 451 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC, 452 SDVTList VTList, ArrayRef<SDValue> OpList) { 453 AddNodeIDOpcode(ID, OpC); 454 AddNodeIDValueTypes(ID, VTList); 455 AddNodeIDOperands(ID, OpList); 456 } 457 458 /// If this is an SDNode with special info, add this info to the NodeID data. 459 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { 460 switch (N->getOpcode()) { 461 case ISD::TargetExternalSymbol: 462 case ISD::ExternalSymbol: 463 case ISD::MCSymbol: 464 llvm_unreachable("Should only be used on nodes with operands"); 465 default: break; // Normal nodes don't need extra info. 466 case ISD::TargetConstant: 467 case ISD::Constant: { 468 const ConstantSDNode *C = cast<ConstantSDNode>(N); 469 ID.AddPointer(C->getConstantIntValue()); 470 ID.AddBoolean(C->isOpaque()); 471 break; 472 } 473 case ISD::TargetConstantFP: 474 case ISD::ConstantFP: 475 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue()); 476 break; 477 case ISD::TargetGlobalAddress: 478 case ISD::GlobalAddress: 479 case ISD::TargetGlobalTLSAddress: 480 case ISD::GlobalTLSAddress: { 481 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 482 ID.AddPointer(GA->getGlobal()); 483 ID.AddInteger(GA->getOffset()); 484 ID.AddInteger(GA->getTargetFlags()); 485 break; 486 } 487 case ISD::BasicBlock: 488 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock()); 489 break; 490 case ISD::Register: 491 ID.AddInteger(cast<RegisterSDNode>(N)->getReg()); 492 break; 493 case ISD::RegisterMask: 494 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask()); 495 break; 496 case ISD::SRCVALUE: 497 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue()); 498 break; 499 case ISD::FrameIndex: 500 case ISD::TargetFrameIndex: 501 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex()); 502 break; 503 case ISD::JumpTable: 504 case ISD::TargetJumpTable: 505 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex()); 506 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags()); 507 break; 508 case ISD::ConstantPool: 509 case ISD::TargetConstantPool: { 510 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N); 511 ID.AddInteger(CP->getAlignment()); 512 ID.AddInteger(CP->getOffset()); 513 if (CP->isMachineConstantPoolEntry()) 514 CP->getMachineCPVal()->addSelectionDAGCSEId(ID); 515 else 516 ID.AddPointer(CP->getConstVal()); 517 ID.AddInteger(CP->getTargetFlags()); 518 break; 519 } 520 case ISD::TargetIndex: { 521 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N); 522 ID.AddInteger(TI->getIndex()); 523 ID.AddInteger(TI->getOffset()); 524 ID.AddInteger(TI->getTargetFlags()); 525 break; 526 } 527 case ISD::LOAD: { 528 const LoadSDNode *LD = cast<LoadSDNode>(N); 529 ID.AddInteger(LD->getMemoryVT().getRawBits()); 530 ID.AddInteger(LD->getRawSubclassData()); 531 ID.AddInteger(LD->getPointerInfo().getAddrSpace()); 532 break; 533 } 534 case ISD::STORE: { 535 const StoreSDNode *ST = cast<StoreSDNode>(N); 536 ID.AddInteger(ST->getMemoryVT().getRawBits()); 537 ID.AddInteger(ST->getRawSubclassData()); 538 ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 539 break; 540 } 541 case ISD::MLOAD: { 542 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N); 543 ID.AddInteger(MLD->getMemoryVT().getRawBits()); 544 ID.AddInteger(MLD->getRawSubclassData()); 545 ID.AddInteger(MLD->getPointerInfo().getAddrSpace()); 546 break; 547 } 548 case ISD::MSTORE: { 549 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N); 550 ID.AddInteger(MST->getMemoryVT().getRawBits()); 551 ID.AddInteger(MST->getRawSubclassData()); 552 ID.AddInteger(MST->getPointerInfo().getAddrSpace()); 553 break; 554 } 555 case ISD::MGATHER: { 556 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N); 557 ID.AddInteger(MG->getMemoryVT().getRawBits()); 558 ID.AddInteger(MG->getRawSubclassData()); 559 ID.AddInteger(MG->getPointerInfo().getAddrSpace()); 560 break; 561 } 562 case ISD::MSCATTER: { 563 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N); 564 ID.AddInteger(MS->getMemoryVT().getRawBits()); 565 ID.AddInteger(MS->getRawSubclassData()); 566 ID.AddInteger(MS->getPointerInfo().getAddrSpace()); 567 break; 568 } 569 case ISD::ATOMIC_CMP_SWAP: 570 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 571 case ISD::ATOMIC_SWAP: 572 case ISD::ATOMIC_LOAD_ADD: 573 case ISD::ATOMIC_LOAD_SUB: 574 case ISD::ATOMIC_LOAD_AND: 575 case ISD::ATOMIC_LOAD_CLR: 576 case ISD::ATOMIC_LOAD_OR: 577 case ISD::ATOMIC_LOAD_XOR: 578 case ISD::ATOMIC_LOAD_NAND: 579 case ISD::ATOMIC_LOAD_MIN: 580 case ISD::ATOMIC_LOAD_MAX: 581 case ISD::ATOMIC_LOAD_UMIN: 582 case ISD::ATOMIC_LOAD_UMAX: 583 case ISD::ATOMIC_LOAD: 584 case ISD::ATOMIC_STORE: { 585 const AtomicSDNode *AT = cast<AtomicSDNode>(N); 586 ID.AddInteger(AT->getMemoryVT().getRawBits()); 587 ID.AddInteger(AT->getRawSubclassData()); 588 ID.AddInteger(AT->getPointerInfo().getAddrSpace()); 589 break; 590 } 591 case ISD::PREFETCH: { 592 const MemSDNode *PF = cast<MemSDNode>(N); 593 ID.AddInteger(PF->getPointerInfo().getAddrSpace()); 594 break; 595 } 596 case ISD::VECTOR_SHUFFLE: { 597 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 598 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements(); 599 i != e; ++i) 600 ID.AddInteger(SVN->getMaskElt(i)); 601 break; 602 } 603 case ISD::TargetBlockAddress: 604 case ISD::BlockAddress: { 605 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N); 606 ID.AddPointer(BA->getBlockAddress()); 607 ID.AddInteger(BA->getOffset()); 608 ID.AddInteger(BA->getTargetFlags()); 609 break; 610 } 611 } // end switch (N->getOpcode()) 612 613 // Target specific memory nodes could also have address spaces to check. 614 if (N->isTargetMemoryOpcode()) 615 ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace()); 616 } 617 618 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID 619 /// data. 620 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { 621 AddNodeIDOpcode(ID, N->getOpcode()); 622 // Add the return value info. 623 AddNodeIDValueTypes(ID, N->getVTList()); 624 // Add the operand info. 625 AddNodeIDOperands(ID, N->ops()); 626 627 // Handle SDNode leafs with special info. 628 AddNodeIDCustom(ID, N); 629 } 630 631 //===----------------------------------------------------------------------===// 632 // SelectionDAG Class 633 //===----------------------------------------------------------------------===// 634 635 /// doNotCSE - Return true if CSE should not be performed for this node. 636 static bool doNotCSE(SDNode *N) { 637 if (N->getValueType(0) == MVT::Glue) 638 return true; // Never CSE anything that produces a flag. 639 640 switch (N->getOpcode()) { 641 default: break; 642 case ISD::HANDLENODE: 643 case ISD::EH_LABEL: 644 return true; // Never CSE these nodes. 645 } 646 647 // Check that remaining values produced are not flags. 648 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) 649 if (N->getValueType(i) == MVT::Glue) 650 return true; // Never CSE anything that produces a flag. 651 652 return false; 653 } 654 655 /// RemoveDeadNodes - This method deletes all unreachable nodes in the 656 /// SelectionDAG. 657 void SelectionDAG::RemoveDeadNodes() { 658 // Create a dummy node (which is not added to allnodes), that adds a reference 659 // to the root node, preventing it from being deleted. 660 HandleSDNode Dummy(getRoot()); 661 662 SmallVector<SDNode*, 128> DeadNodes; 663 664 // Add all obviously-dead nodes to the DeadNodes worklist. 665 for (SDNode &Node : allnodes()) 666 if (Node.use_empty()) 667 DeadNodes.push_back(&Node); 668 669 RemoveDeadNodes(DeadNodes); 670 671 // If the root changed (e.g. it was a dead load, update the root). 672 setRoot(Dummy.getValue()); 673 } 674 675 /// RemoveDeadNodes - This method deletes the unreachable nodes in the 676 /// given list, and any nodes that become unreachable as a result. 677 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) { 678 679 // Process the worklist, deleting the nodes and adding their uses to the 680 // worklist. 681 while (!DeadNodes.empty()) { 682 SDNode *N = DeadNodes.pop_back_val(); 683 // Skip to next node if we've already managed to delete the node. This could 684 // happen if replacing a node causes a node previously added to the node to 685 // be deleted. 686 if (N->getOpcode() == ISD::DELETED_NODE) 687 continue; 688 689 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 690 DUL->NodeDeleted(N, nullptr); 691 692 // Take the node out of the appropriate CSE map. 693 RemoveNodeFromCSEMaps(N); 694 695 // Next, brutally remove the operand list. This is safe to do, as there are 696 // no cycles in the graph. 697 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 698 SDUse &Use = *I++; 699 SDNode *Operand = Use.getNode(); 700 Use.set(SDValue()); 701 702 // Now that we removed this operand, see if there are no uses of it left. 703 if (Operand->use_empty()) 704 DeadNodes.push_back(Operand); 705 } 706 707 DeallocateNode(N); 708 } 709 } 710 711 void SelectionDAG::RemoveDeadNode(SDNode *N){ 712 SmallVector<SDNode*, 16> DeadNodes(1, N); 713 714 // Create a dummy node that adds a reference to the root node, preventing 715 // it from being deleted. (This matters if the root is an operand of the 716 // dead node.) 717 HandleSDNode Dummy(getRoot()); 718 719 RemoveDeadNodes(DeadNodes); 720 } 721 722 void SelectionDAG::DeleteNode(SDNode *N) { 723 // First take this out of the appropriate CSE map. 724 RemoveNodeFromCSEMaps(N); 725 726 // Finally, remove uses due to operands of this node, remove from the 727 // AllNodes list, and delete the node. 728 DeleteNodeNotInCSEMaps(N); 729 } 730 731 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { 732 assert(N->getIterator() != AllNodes.begin() && 733 "Cannot delete the entry node!"); 734 assert(N->use_empty() && "Cannot delete a node that is not dead!"); 735 736 // Drop all of the operands and decrement used node's use counts. 737 N->DropOperands(); 738 739 DeallocateNode(N); 740 } 741 742 void SDDbgInfo::erase(const SDNode *Node) { 743 DbgValMapType::iterator I = DbgValMap.find(Node); 744 if (I == DbgValMap.end()) 745 return; 746 for (auto &Val: I->second) 747 Val->setIsInvalidated(); 748 DbgValMap.erase(I); 749 } 750 751 void SelectionDAG::DeallocateNode(SDNode *N) { 752 // If we have operands, deallocate them. 753 removeOperands(N); 754 755 NodeAllocator.Deallocate(AllNodes.remove(N)); 756 757 // Set the opcode to DELETED_NODE to help catch bugs when node 758 // memory is reallocated. 759 // FIXME: There are places in SDag that have grown a dependency on the opcode 760 // value in the released node. 761 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType)); 762 N->NodeType = ISD::DELETED_NODE; 763 764 // If any of the SDDbgValue nodes refer to this SDNode, invalidate 765 // them and forget about that node. 766 DbgInfo->erase(N); 767 } 768 769 #ifndef NDEBUG 770 /// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid. 771 static void VerifySDNode(SDNode *N) { 772 switch (N->getOpcode()) { 773 default: 774 break; 775 case ISD::BUILD_PAIR: { 776 EVT VT = N->getValueType(0); 777 assert(N->getNumValues() == 1 && "Too many results!"); 778 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && 779 "Wrong return type!"); 780 assert(N->getNumOperands() == 2 && "Wrong number of operands!"); 781 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && 782 "Mismatched operand types!"); 783 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && 784 "Wrong operand type!"); 785 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && 786 "Wrong return type size"); 787 break; 788 } 789 case ISD::BUILD_VECTOR: { 790 assert(N->getNumValues() == 1 && "Too many results!"); 791 assert(N->getValueType(0).isVector() && "Wrong return type!"); 792 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && 793 "Wrong number of operands!"); 794 EVT EltVT = N->getValueType(0).getVectorElementType(); 795 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { 796 assert((I->getValueType() == EltVT || 797 (EltVT.isInteger() && I->getValueType().isInteger() && 798 EltVT.bitsLE(I->getValueType()))) && 799 "Wrong operand type!"); 800 assert(I->getValueType() == N->getOperand(0).getValueType() && 801 "Operands must all have the same type"); 802 } 803 break; 804 } 805 } 806 } 807 #endif // NDEBUG 808 809 /// Insert a newly allocated node into the DAG. 810 /// 811 /// Handles insertion into the all nodes list and CSE map, as well as 812 /// verification and other common operations when a new node is allocated. 813 void SelectionDAG::InsertNode(SDNode *N) { 814 AllNodes.push_back(N); 815 #ifndef NDEBUG 816 N->PersistentId = NextPersistentId++; 817 VerifySDNode(N); 818 #endif 819 } 820 821 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that 822 /// correspond to it. This is useful when we're about to delete or repurpose 823 /// the node. We don't want future request for structurally identical nodes 824 /// to return N anymore. 825 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { 826 bool Erased = false; 827 switch (N->getOpcode()) { 828 case ISD::HANDLENODE: return false; // noop. 829 case ISD::CONDCODE: 830 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && 831 "Cond code doesn't exist!"); 832 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr; 833 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr; 834 break; 835 case ISD::ExternalSymbol: 836 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol()); 837 break; 838 case ISD::TargetExternalSymbol: { 839 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N); 840 Erased = TargetExternalSymbols.erase( 841 std::pair<std::string,unsigned char>(ESN->getSymbol(), 842 ESN->getTargetFlags())); 843 break; 844 } 845 case ISD::MCSymbol: { 846 auto *MCSN = cast<MCSymbolSDNode>(N); 847 Erased = MCSymbols.erase(MCSN->getMCSymbol()); 848 break; 849 } 850 case ISD::VALUETYPE: { 851 EVT VT = cast<VTSDNode>(N)->getVT(); 852 if (VT.isExtended()) { 853 Erased = ExtendedValueTypeNodes.erase(VT); 854 } else { 855 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr; 856 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr; 857 } 858 break; 859 } 860 default: 861 // Remove it from the CSE Map. 862 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); 863 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); 864 Erased = CSEMap.RemoveNode(N); 865 break; 866 } 867 #ifndef NDEBUG 868 // Verify that the node was actually in one of the CSE maps, unless it has a 869 // flag result (which cannot be CSE'd) or is one of the special cases that are 870 // not subject to CSE. 871 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && 872 !N->isMachineOpcode() && !doNotCSE(N)) { 873 N->dump(this); 874 dbgs() << "\n"; 875 llvm_unreachable("Node is not in map!"); 876 } 877 #endif 878 return Erased; 879 } 880 881 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE 882 /// maps and modified in place. Add it back to the CSE maps, unless an identical 883 /// node already exists, in which case transfer all its users to the existing 884 /// node. This transfer can potentially trigger recursive merging. 885 void 886 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) { 887 // For node types that aren't CSE'd, just act as if no identical node 888 // already exists. 889 if (!doNotCSE(N)) { 890 SDNode *Existing = CSEMap.GetOrInsertNode(N); 891 if (Existing != N) { 892 // If there was already an existing matching node, use ReplaceAllUsesWith 893 // to replace the dead one with the existing one. This can cause 894 // recursive merging of other unrelated nodes down the line. 895 ReplaceAllUsesWith(N, Existing); 896 897 // N is now dead. Inform the listeners and delete it. 898 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 899 DUL->NodeDeleted(N, Existing); 900 DeleteNodeNotInCSEMaps(N); 901 return; 902 } 903 } 904 905 // If the node doesn't already exist, we updated it. Inform listeners. 906 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 907 DUL->NodeUpdated(N); 908 } 909 910 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands 911 /// were replaced with those specified. If this node is never memoized, 912 /// return null, otherwise return a pointer to the slot it would take. If a 913 /// node already exists with these operands, the slot will be non-null. 914 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, 915 void *&InsertPos) { 916 if (doNotCSE(N)) 917 return nullptr; 918 919 SDValue Ops[] = { Op }; 920 FoldingSetNodeID ID; 921 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); 922 AddNodeIDCustom(ID, N); 923 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); 924 if (Node) 925 Node->intersectFlagsWith(N->getFlags()); 926 return Node; 927 } 928 929 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands 930 /// were replaced with those specified. If this node is never memoized, 931 /// return null, otherwise return a pointer to the slot it would take. If a 932 /// node already exists with these operands, the slot will be non-null. 933 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 934 SDValue Op1, SDValue Op2, 935 void *&InsertPos) { 936 if (doNotCSE(N)) 937 return nullptr; 938 939 SDValue Ops[] = { Op1, Op2 }; 940 FoldingSetNodeID ID; 941 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); 942 AddNodeIDCustom(ID, N); 943 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); 944 if (Node) 945 Node->intersectFlagsWith(N->getFlags()); 946 return Node; 947 } 948 949 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands 950 /// were replaced with those specified. If this node is never memoized, 951 /// return null, otherwise return a pointer to the slot it would take. If a 952 /// node already exists with these operands, the slot will be non-null. 953 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops, 954 void *&InsertPos) { 955 if (doNotCSE(N)) 956 return nullptr; 957 958 FoldingSetNodeID ID; 959 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); 960 AddNodeIDCustom(ID, N); 961 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); 962 if (Node) 963 Node->intersectFlagsWith(N->getFlags()); 964 return Node; 965 } 966 967 unsigned SelectionDAG::getEVTAlignment(EVT VT) const { 968 Type *Ty = VT == MVT::iPTR ? 969 PointerType::get(Type::getInt8Ty(*getContext()), 0) : 970 VT.getTypeForEVT(*getContext()); 971 972 return getDataLayout().getABITypeAlignment(Ty); 973 } 974 975 // EntryNode could meaningfully have debug info if we can find it... 976 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) 977 : TM(tm), OptLevel(OL), 978 EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)), 979 Root(getEntryNode()) { 980 InsertNode(&EntryNode); 981 DbgInfo = new SDDbgInfo(); 982 } 983 984 void SelectionDAG::init(MachineFunction &NewMF, 985 OptimizationRemarkEmitter &NewORE, 986 Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, 987 DivergenceAnalysis * Divergence) { 988 MF = &NewMF; 989 SDAGISelPass = PassPtr; 990 ORE = &NewORE; 991 TLI = getSubtarget().getTargetLowering(); 992 TSI = getSubtarget().getSelectionDAGInfo(); 993 LibInfo = LibraryInfo; 994 Context = &MF->getFunction().getContext(); 995 DA = Divergence; 996 } 997 998 SelectionDAG::~SelectionDAG() { 999 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners"); 1000 allnodes_clear(); 1001 OperandRecycler.clear(OperandAllocator); 1002 delete DbgInfo; 1003 } 1004 1005 void SelectionDAG::allnodes_clear() { 1006 assert(&*AllNodes.begin() == &EntryNode); 1007 AllNodes.remove(AllNodes.begin()); 1008 while (!AllNodes.empty()) 1009 DeallocateNode(&AllNodes.front()); 1010 #ifndef NDEBUG 1011 NextPersistentId = 0; 1012 #endif 1013 } 1014 1015 SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID, 1016 void *&InsertPos) { 1017 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 1018 if (N) { 1019 switch (N->getOpcode()) { 1020 default: break; 1021 case ISD::Constant: 1022 case ISD::ConstantFP: 1023 llvm_unreachable("Querying for Constant and ConstantFP nodes requires " 1024 "debug location. Use another overload."); 1025 } 1026 } 1027 return N; 1028 } 1029 1030 SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID, 1031 const SDLoc &DL, void *&InsertPos) { 1032 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 1033 if (N) { 1034 switch (N->getOpcode()) { 1035 case ISD::Constant: 1036 case ISD::ConstantFP: 1037 // Erase debug location from the node if the node is used at several 1038 // different places. Do not propagate one location to all uses as it 1039 // will cause a worse single stepping debugging experience. 1040 if (N->getDebugLoc() != DL.getDebugLoc()) 1041 N->setDebugLoc(DebugLoc()); 1042 break; 1043 default: 1044 // When the node's point of use is located earlier in the instruction 1045 // sequence than its prior point of use, update its debug info to the 1046 // earlier location. 1047 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder()) 1048 N->setDebugLoc(DL.getDebugLoc()); 1049 break; 1050 } 1051 } 1052 return N; 1053 } 1054 1055 void SelectionDAG::clear() { 1056 allnodes_clear(); 1057 OperandRecycler.clear(OperandAllocator); 1058 OperandAllocator.Reset(); 1059 CSEMap.clear(); 1060 1061 ExtendedValueTypeNodes.clear(); 1062 ExternalSymbols.clear(); 1063 TargetExternalSymbols.clear(); 1064 MCSymbols.clear(); 1065 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), 1066 static_cast<CondCodeSDNode*>(nullptr)); 1067 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), 1068 static_cast<SDNode*>(nullptr)); 1069 1070 EntryNode.UseList = nullptr; 1071 InsertNode(&EntryNode); 1072 Root = getEntryNode(); 1073 DbgInfo->clear(); 1074 } 1075 1076 SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) { 1077 return VT.bitsGT(Op.getValueType()) 1078 ? getNode(ISD::FP_EXTEND, DL, VT, Op) 1079 : getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL)); 1080 } 1081 1082 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { 1083 return VT.bitsGT(Op.getValueType()) ? 1084 getNode(ISD::ANY_EXTEND, DL, VT, Op) : 1085 getNode(ISD::TRUNCATE, DL, VT, Op); 1086 } 1087 1088 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { 1089 return VT.bitsGT(Op.getValueType()) ? 1090 getNode(ISD::SIGN_EXTEND, DL, VT, Op) : 1091 getNode(ISD::TRUNCATE, DL, VT, Op); 1092 } 1093 1094 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { 1095 return VT.bitsGT(Op.getValueType()) ? 1096 getNode(ISD::ZERO_EXTEND, DL, VT, Op) : 1097 getNode(ISD::TRUNCATE, DL, VT, Op); 1098 } 1099 1100 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, 1101 EVT OpVT) { 1102 if (VT.bitsLE(Op.getValueType())) 1103 return getNode(ISD::TRUNCATE, SL, VT, Op); 1104 1105 TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT); 1106 return getNode(TLI->getExtendForContent(BType), SL, VT, Op); 1107 } 1108 1109 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) { 1110 assert(!VT.isVector() && 1111 "getZeroExtendInReg should use the vector element type instead of " 1112 "the vector type!"); 1113 if (Op.getValueType().getScalarType() == VT) return Op; 1114 unsigned BitWidth = Op.getScalarValueSizeInBits(); 1115 APInt Imm = APInt::getLowBitsSet(BitWidth, 1116 VT.getSizeInBits()); 1117 return getNode(ISD::AND, DL, Op.getValueType(), Op, 1118 getConstant(Imm, DL, Op.getValueType())); 1119 } 1120 1121 SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL, 1122 EVT VT) { 1123 assert(VT.isVector() && "This DAG node is restricted to vector types."); 1124 assert(VT.getSizeInBits() == Op.getValueSizeInBits() && 1125 "The sizes of the input and result must match in order to perform the " 1126 "extend in-register."); 1127 assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() && 1128 "The destination vector type must have fewer lanes than the input."); 1129 return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op); 1130 } 1131 1132 SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, const SDLoc &DL, 1133 EVT VT) { 1134 assert(VT.isVector() && "This DAG node is restricted to vector types."); 1135 assert(VT.getSizeInBits() == Op.getValueSizeInBits() && 1136 "The sizes of the input and result must match in order to perform the " 1137 "extend in-register."); 1138 assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() && 1139 "The destination vector type must have fewer lanes than the input."); 1140 return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op); 1141 } 1142 1143 SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, const SDLoc &DL, 1144 EVT VT) { 1145 assert(VT.isVector() && "This DAG node is restricted to vector types."); 1146 assert(VT.getSizeInBits() == Op.getValueSizeInBits() && 1147 "The sizes of the input and result must match in order to perform the " 1148 "extend in-register."); 1149 assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() && 1150 "The destination vector type must have fewer lanes than the input."); 1151 return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op); 1152 } 1153 1154 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1). 1155 SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) { 1156 EVT EltVT = VT.getScalarType(); 1157 SDValue NegOne = 1158 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT); 1159 return getNode(ISD::XOR, DL, VT, Val, NegOne); 1160 } 1161 1162 SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) { 1163 SDValue TrueValue = getBoolConstant(true, DL, VT, VT); 1164 return getNode(ISD::XOR, DL, VT, Val, TrueValue); 1165 } 1166 1167 SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT, 1168 EVT OpVT) { 1169 if (!V) 1170 return getConstant(0, DL, VT); 1171 1172 switch (TLI->getBooleanContents(OpVT)) { 1173 case TargetLowering::ZeroOrOneBooleanContent: 1174 case TargetLowering::UndefinedBooleanContent: 1175 return getConstant(1, DL, VT); 1176 case TargetLowering::ZeroOrNegativeOneBooleanContent: 1177 return getAllOnesConstant(DL, VT); 1178 } 1179 llvm_unreachable("Unexpected boolean content enum!"); 1180 } 1181 1182 SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT, 1183 bool isT, bool isO) { 1184 EVT EltVT = VT.getScalarType(); 1185 assert((EltVT.getSizeInBits() >= 64 || 1186 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && 1187 "getConstant with a uint64_t value that doesn't fit in the type!"); 1188 return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO); 1189 } 1190 1191 SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT, 1192 bool isT, bool isO) { 1193 return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO); 1194 } 1195 1196 SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL, 1197 EVT VT, bool isT, bool isO) { 1198 assert(VT.isInteger() && "Cannot create FP integer constant!"); 1199 1200 EVT EltVT = VT.getScalarType(); 1201 const ConstantInt *Elt = &Val; 1202 1203 // In some cases the vector type is legal but the element type is illegal and 1204 // needs to be promoted, for example v8i8 on ARM. In this case, promote the 1205 // inserted value (the type does not need to match the vector element type). 1206 // Any extra bits introduced will be truncated away. 1207 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) == 1208 TargetLowering::TypePromoteInteger) { 1209 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); 1210 APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits()); 1211 Elt = ConstantInt::get(*getContext(), NewVal); 1212 } 1213 // In other cases the element type is illegal and needs to be expanded, for 1214 // example v2i64 on MIPS32. In this case, find the nearest legal type, split 1215 // the value into n parts and use a vector type with n-times the elements. 1216 // Then bitcast to the type requested. 1217 // Legalizing constants too early makes the DAGCombiner's job harder so we 1218 // only legalize if the DAG tells us we must produce legal types. 1219 else if (NewNodesMustHaveLegalTypes && VT.isVector() && 1220 TLI->getTypeAction(*getContext(), EltVT) == 1221 TargetLowering::TypeExpandInteger) { 1222 const APInt &NewVal = Elt->getValue(); 1223 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); 1224 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits(); 1225 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits; 1226 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts); 1227 1228 // Check the temporary vector is the correct size. If this fails then 1229 // getTypeToTransformTo() probably returned a type whose size (in bits) 1230 // isn't a power-of-2 factor of the requested type size. 1231 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits()); 1232 1233 SmallVector<SDValue, 2> EltParts; 1234 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) { 1235 EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits) 1236 .zextOrTrunc(ViaEltSizeInBits), DL, 1237 ViaEltVT, isT, isO)); 1238 } 1239 1240 // EltParts is currently in little endian order. If we actually want 1241 // big-endian order then reverse it now. 1242 if (getDataLayout().isBigEndian()) 1243 std::reverse(EltParts.begin(), EltParts.end()); 1244 1245 // The elements must be reversed when the element order is different 1246 // to the endianness of the elements (because the BITCAST is itself a 1247 // vector shuffle in this situation). However, we do not need any code to 1248 // perform this reversal because getConstant() is producing a vector 1249 // splat. 1250 // This situation occurs in MIPS MSA. 1251 1252 SmallVector<SDValue, 8> Ops; 1253 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) 1254 Ops.insert(Ops.end(), EltParts.begin(), EltParts.end()); 1255 1256 SDValue V = getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops)); 1257 return V; 1258 } 1259 1260 assert(Elt->getBitWidth() == EltVT.getSizeInBits() && 1261 "APInt size does not match type size!"); 1262 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; 1263 FoldingSetNodeID ID; 1264 AddNodeIDNode(ID, Opc, getVTList(EltVT), None); 1265 ID.AddPointer(Elt); 1266 ID.AddBoolean(isO); 1267 void *IP = nullptr; 1268 SDNode *N = nullptr; 1269 if ((N = FindNodeOrInsertPos(ID, DL, IP))) 1270 if (!VT.isVector()) 1271 return SDValue(N, 0); 1272 1273 if (!N) { 1274 N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT); 1275 CSEMap.InsertNode(N, IP); 1276 InsertNode(N); 1277 NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this); 1278 } 1279 1280 SDValue Result(N, 0); 1281 if (VT.isVector()) 1282 Result = getSplatBuildVector(VT, DL, Result); 1283 1284 return Result; 1285 } 1286 1287 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL, 1288 bool isTarget) { 1289 return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget); 1290 } 1291 1292 SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT, 1293 bool isTarget) { 1294 return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget); 1295 } 1296 1297 SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL, 1298 EVT VT, bool isTarget) { 1299 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); 1300 1301 EVT EltVT = VT.getScalarType(); 1302 1303 // Do the map lookup using the actual bit pattern for the floating point 1304 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and 1305 // we don't have issues with SNANs. 1306 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; 1307 FoldingSetNodeID ID; 1308 AddNodeIDNode(ID, Opc, getVTList(EltVT), None); 1309 ID.AddPointer(&V); 1310 void *IP = nullptr; 1311 SDNode *N = nullptr; 1312 if ((N = FindNodeOrInsertPos(ID, DL, IP))) 1313 if (!VT.isVector()) 1314 return SDValue(N, 0); 1315 1316 if (!N) { 1317 N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT); 1318 CSEMap.InsertNode(N, IP); 1319 InsertNode(N); 1320 } 1321 1322 SDValue Result(N, 0); 1323 if (VT.isVector()) 1324 Result = getSplatBuildVector(VT, DL, Result); 1325 NewSDValueDbgMsg(Result, "Creating fp constant: ", this); 1326 return Result; 1327 } 1328 1329 SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT, 1330 bool isTarget) { 1331 EVT EltVT = VT.getScalarType(); 1332 if (EltVT == MVT::f32) 1333 return getConstantFP(APFloat((float)Val), DL, VT, isTarget); 1334 else if (EltVT == MVT::f64) 1335 return getConstantFP(APFloat(Val), DL, VT, isTarget); 1336 else if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 || 1337 EltVT == MVT::f16) { 1338 bool Ignored; 1339 APFloat APF = APFloat(Val); 1340 APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, 1341 &Ignored); 1342 return getConstantFP(APF, DL, VT, isTarget); 1343 } else 1344 llvm_unreachable("Unsupported type in getConstantFP"); 1345 } 1346 1347 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, 1348 EVT VT, int64_t Offset, bool isTargetGA, 1349 unsigned char TargetFlags) { 1350 assert((TargetFlags == 0 || isTargetGA) && 1351 "Cannot set target flags on target-independent globals"); 1352 1353 // Truncate (with sign-extension) the offset value to the pointer size. 1354 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType()); 1355 if (BitWidth < 64) 1356 Offset = SignExtend64(Offset, BitWidth); 1357 1358 unsigned Opc; 1359 if (GV->isThreadLocal()) 1360 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; 1361 else 1362 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; 1363 1364 FoldingSetNodeID ID; 1365 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1366 ID.AddPointer(GV); 1367 ID.AddInteger(Offset); 1368 ID.AddInteger(TargetFlags); 1369 void *IP = nullptr; 1370 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) 1371 return SDValue(E, 0); 1372 1373 auto *N = newSDNode<GlobalAddressSDNode>( 1374 Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags); 1375 CSEMap.InsertNode(N, IP); 1376 InsertNode(N); 1377 return SDValue(N, 0); 1378 } 1379 1380 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { 1381 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; 1382 FoldingSetNodeID ID; 1383 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1384 ID.AddInteger(FI); 1385 void *IP = nullptr; 1386 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1387 return SDValue(E, 0); 1388 1389 auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget); 1390 CSEMap.InsertNode(N, IP); 1391 InsertNode(N); 1392 return SDValue(N, 0); 1393 } 1394 1395 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, 1396 unsigned char TargetFlags) { 1397 assert((TargetFlags == 0 || isTarget) && 1398 "Cannot set target flags on target-independent jump tables"); 1399 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; 1400 FoldingSetNodeID ID; 1401 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1402 ID.AddInteger(JTI); 1403 ID.AddInteger(TargetFlags); 1404 void *IP = nullptr; 1405 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1406 return SDValue(E, 0); 1407 1408 auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags); 1409 CSEMap.InsertNode(N, IP); 1410 InsertNode(N); 1411 return SDValue(N, 0); 1412 } 1413 1414 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, 1415 unsigned Alignment, int Offset, 1416 bool isTarget, 1417 unsigned char TargetFlags) { 1418 assert((TargetFlags == 0 || isTarget) && 1419 "Cannot set target flags on target-independent globals"); 1420 if (Alignment == 0) 1421 Alignment = MF->getFunction().optForSize() 1422 ? getDataLayout().getABITypeAlignment(C->getType()) 1423 : getDataLayout().getPrefTypeAlignment(C->getType()); 1424 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1425 FoldingSetNodeID ID; 1426 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1427 ID.AddInteger(Alignment); 1428 ID.AddInteger(Offset); 1429 ID.AddPointer(C); 1430 ID.AddInteger(TargetFlags); 1431 void *IP = nullptr; 1432 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1433 return SDValue(E, 0); 1434 1435 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment, 1436 TargetFlags); 1437 CSEMap.InsertNode(N, IP); 1438 InsertNode(N); 1439 return SDValue(N, 0); 1440 } 1441 1442 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, 1443 unsigned Alignment, int Offset, 1444 bool isTarget, 1445 unsigned char TargetFlags) { 1446 assert((TargetFlags == 0 || isTarget) && 1447 "Cannot set target flags on target-independent globals"); 1448 if (Alignment == 0) 1449 Alignment = getDataLayout().getPrefTypeAlignment(C->getType()); 1450 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1451 FoldingSetNodeID ID; 1452 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1453 ID.AddInteger(Alignment); 1454 ID.AddInteger(Offset); 1455 C->addSelectionDAGCSEId(ID); 1456 ID.AddInteger(TargetFlags); 1457 void *IP = nullptr; 1458 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1459 return SDValue(E, 0); 1460 1461 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment, 1462 TargetFlags); 1463 CSEMap.InsertNode(N, IP); 1464 InsertNode(N); 1465 return SDValue(N, 0); 1466 } 1467 1468 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset, 1469 unsigned char TargetFlags) { 1470 FoldingSetNodeID ID; 1471 AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None); 1472 ID.AddInteger(Index); 1473 ID.AddInteger(Offset); 1474 ID.AddInteger(TargetFlags); 1475 void *IP = nullptr; 1476 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1477 return SDValue(E, 0); 1478 1479 auto *N = newSDNode<TargetIndexSDNode>(Index, VT, Offset, TargetFlags); 1480 CSEMap.InsertNode(N, IP); 1481 InsertNode(N); 1482 return SDValue(N, 0); 1483 } 1484 1485 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { 1486 FoldingSetNodeID ID; 1487 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None); 1488 ID.AddPointer(MBB); 1489 void *IP = nullptr; 1490 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1491 return SDValue(E, 0); 1492 1493 auto *N = newSDNode<BasicBlockSDNode>(MBB); 1494 CSEMap.InsertNode(N, IP); 1495 InsertNode(N); 1496 return SDValue(N, 0); 1497 } 1498 1499 SDValue SelectionDAG::getValueType(EVT VT) { 1500 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >= 1501 ValueTypeNodes.size()) 1502 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1); 1503 1504 SDNode *&N = VT.isExtended() ? 1505 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; 1506 1507 if (N) return SDValue(N, 0); 1508 N = newSDNode<VTSDNode>(VT); 1509 InsertNode(N); 1510 return SDValue(N, 0); 1511 } 1512 1513 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { 1514 SDNode *&N = ExternalSymbols[Sym]; 1515 if (N) return SDValue(N, 0); 1516 N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT); 1517 InsertNode(N); 1518 return SDValue(N, 0); 1519 } 1520 1521 SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) { 1522 SDNode *&N = MCSymbols[Sym]; 1523 if (N) 1524 return SDValue(N, 0); 1525 N = newSDNode<MCSymbolSDNode>(Sym, VT); 1526 InsertNode(N); 1527 return SDValue(N, 0); 1528 } 1529 1530 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, 1531 unsigned char TargetFlags) { 1532 SDNode *&N = 1533 TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym, 1534 TargetFlags)]; 1535 if (N) return SDValue(N, 0); 1536 N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT); 1537 InsertNode(N); 1538 return SDValue(N, 0); 1539 } 1540 1541 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { 1542 if ((unsigned)Cond >= CondCodeNodes.size()) 1543 CondCodeNodes.resize(Cond+1); 1544 1545 if (!CondCodeNodes[Cond]) { 1546 auto *N = newSDNode<CondCodeSDNode>(Cond); 1547 CondCodeNodes[Cond] = N; 1548 InsertNode(N); 1549 } 1550 1551 return SDValue(CondCodeNodes[Cond], 0); 1552 } 1553 1554 /// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that 1555 /// point at N1 to point at N2 and indices that point at N2 to point at N1. 1556 static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) { 1557 std::swap(N1, N2); 1558 ShuffleVectorSDNode::commuteMask(M); 1559 } 1560 1561 SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, 1562 SDValue N2, ArrayRef<int> Mask) { 1563 assert(VT.getVectorNumElements() == Mask.size() && 1564 "Must have the same number of vector elements as mask elements!"); 1565 assert(VT == N1.getValueType() && VT == N2.getValueType() && 1566 "Invalid VECTOR_SHUFFLE"); 1567 1568 // Canonicalize shuffle undef, undef -> undef 1569 if (N1.isUndef() && N2.isUndef()) 1570 return getUNDEF(VT); 1571 1572 // Validate that all indices in Mask are within the range of the elements 1573 // input to the shuffle. 1574 int NElts = Mask.size(); 1575 assert(llvm::all_of(Mask, 1576 [&](int M) { return M < (NElts * 2) && M >= -1; }) && 1577 "Index out of range"); 1578 1579 // Copy the mask so we can do any needed cleanup. 1580 SmallVector<int, 8> MaskVec(Mask.begin(), Mask.end()); 1581 1582 // Canonicalize shuffle v, v -> v, undef 1583 if (N1 == N2) { 1584 N2 = getUNDEF(VT); 1585 for (int i = 0; i != NElts; ++i) 1586 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts; 1587 } 1588 1589 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. 1590 if (N1.isUndef()) 1591 commuteShuffle(N1, N2, MaskVec); 1592 1593 if (TLI->hasVectorBlend()) { 1594 // If shuffling a splat, try to blend the splat instead. We do this here so 1595 // that even when this arises during lowering we don't have to re-handle it. 1596 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) { 1597 BitVector UndefElements; 1598 SDValue Splat = BV->getSplatValue(&UndefElements); 1599 if (!Splat) 1600 return; 1601 1602 for (int i = 0; i < NElts; ++i) { 1603 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts)) 1604 continue; 1605 1606 // If this input comes from undef, mark it as such. 1607 if (UndefElements[MaskVec[i] - Offset]) { 1608 MaskVec[i] = -1; 1609 continue; 1610 } 1611 1612 // If we can blend a non-undef lane, use that instead. 1613 if (!UndefElements[i]) 1614 MaskVec[i] = i + Offset; 1615 } 1616 }; 1617 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1)) 1618 BlendSplat(N1BV, 0); 1619 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2)) 1620 BlendSplat(N2BV, NElts); 1621 } 1622 1623 // Canonicalize all index into lhs, -> shuffle lhs, undef 1624 // Canonicalize all index into rhs, -> shuffle rhs, undef 1625 bool AllLHS = true, AllRHS = true; 1626 bool N2Undef = N2.isUndef(); 1627 for (int i = 0; i != NElts; ++i) { 1628 if (MaskVec[i] >= NElts) { 1629 if (N2Undef) 1630 MaskVec[i] = -1; 1631 else 1632 AllLHS = false; 1633 } else if (MaskVec[i] >= 0) { 1634 AllRHS = false; 1635 } 1636 } 1637 if (AllLHS && AllRHS) 1638 return getUNDEF(VT); 1639 if (AllLHS && !N2Undef) 1640 N2 = getUNDEF(VT); 1641 if (AllRHS) { 1642 N1 = getUNDEF(VT); 1643 commuteShuffle(N1, N2, MaskVec); 1644 } 1645 // Reset our undef status after accounting for the mask. 1646 N2Undef = N2.isUndef(); 1647 // Re-check whether both sides ended up undef. 1648 if (N1.isUndef() && N2Undef) 1649 return getUNDEF(VT); 1650 1651 // If Identity shuffle return that node. 1652 bool Identity = true, AllSame = true; 1653 for (int i = 0; i != NElts; ++i) { 1654 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false; 1655 if (MaskVec[i] != MaskVec[0]) AllSame = false; 1656 } 1657 if (Identity && NElts) 1658 return N1; 1659 1660 // Shuffling a constant splat doesn't change the result. 1661 if (N2Undef) { 1662 SDValue V = N1; 1663 1664 // Look through any bitcasts. We check that these don't change the number 1665 // (and size) of elements and just changes their types. 1666 while (V.getOpcode() == ISD::BITCAST) 1667 V = V->getOperand(0); 1668 1669 // A splat should always show up as a build vector node. 1670 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) { 1671 BitVector UndefElements; 1672 SDValue Splat = BV->getSplatValue(&UndefElements); 1673 // If this is a splat of an undef, shuffling it is also undef. 1674 if (Splat && Splat.isUndef()) 1675 return getUNDEF(VT); 1676 1677 bool SameNumElts = 1678 V.getValueType().getVectorNumElements() == VT.getVectorNumElements(); 1679 1680 // We only have a splat which can skip shuffles if there is a splatted 1681 // value and no undef lanes rearranged by the shuffle. 1682 if (Splat && UndefElements.none()) { 1683 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the 1684 // number of elements match or the value splatted is a zero constant. 1685 if (SameNumElts) 1686 return N1; 1687 if (auto *C = dyn_cast<ConstantSDNode>(Splat)) 1688 if (C->isNullValue()) 1689 return N1; 1690 } 1691 1692 // If the shuffle itself creates a splat, build the vector directly. 1693 if (AllSame && SameNumElts) { 1694 EVT BuildVT = BV->getValueType(0); 1695 const SDValue &Splatted = BV->getOperand(MaskVec[0]); 1696 SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted); 1697 1698 // We may have jumped through bitcasts, so the type of the 1699 // BUILD_VECTOR may not match the type of the shuffle. 1700 if (BuildVT != VT) 1701 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV); 1702 return NewBV; 1703 } 1704 } 1705 } 1706 1707 FoldingSetNodeID ID; 1708 SDValue Ops[2] = { N1, N2 }; 1709 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops); 1710 for (int i = 0; i != NElts; ++i) 1711 ID.AddInteger(MaskVec[i]); 1712 1713 void* IP = nullptr; 1714 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) 1715 return SDValue(E, 0); 1716 1717 // Allocate the mask array for the node out of the BumpPtrAllocator, since 1718 // SDNode doesn't have access to it. This memory will be "leaked" when 1719 // the node is deallocated, but recovered when the NodeAllocator is released. 1720 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts); 1721 std::copy(MaskVec.begin(), MaskVec.end(), MaskAlloc); 1722 1723 auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(), 1724 dl.getDebugLoc(), MaskAlloc); 1725 createOperands(N, Ops); 1726 1727 CSEMap.InsertNode(N, IP); 1728 InsertNode(N); 1729 SDValue V = SDValue(N, 0); 1730 NewSDValueDbgMsg(V, "Creating new node: ", this); 1731 return V; 1732 } 1733 1734 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) { 1735 EVT VT = SV.getValueType(0); 1736 SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end()); 1737 ShuffleVectorSDNode::commuteMask(MaskVec); 1738 1739 SDValue Op0 = SV.getOperand(0); 1740 SDValue Op1 = SV.getOperand(1); 1741 return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec); 1742 } 1743 1744 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { 1745 FoldingSetNodeID ID; 1746 AddNodeIDNode(ID, ISD::Register, getVTList(VT), None); 1747 ID.AddInteger(RegNo); 1748 void *IP = nullptr; 1749 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1750 return SDValue(E, 0); 1751 1752 auto *N = newSDNode<RegisterSDNode>(RegNo, VT); 1753 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, DA); 1754 CSEMap.InsertNode(N, IP); 1755 InsertNode(N); 1756 return SDValue(N, 0); 1757 } 1758 1759 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) { 1760 FoldingSetNodeID ID; 1761 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None); 1762 ID.AddPointer(RegMask); 1763 void *IP = nullptr; 1764 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1765 return SDValue(E, 0); 1766 1767 auto *N = newSDNode<RegisterMaskSDNode>(RegMask); 1768 CSEMap.InsertNode(N, IP); 1769 InsertNode(N); 1770 return SDValue(N, 0); 1771 } 1772 1773 SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root, 1774 MCSymbol *Label) { 1775 return getLabelNode(ISD::EH_LABEL, dl, Root, Label); 1776 } 1777 1778 SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl, 1779 SDValue Root, MCSymbol *Label) { 1780 FoldingSetNodeID ID; 1781 SDValue Ops[] = { Root }; 1782 AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops); 1783 ID.AddPointer(Label); 1784 void *IP = nullptr; 1785 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1786 return SDValue(E, 0); 1787 1788 auto *N = newSDNode<LabelSDNode>(dl.getIROrder(), dl.getDebugLoc(), Label); 1789 createOperands(N, Ops); 1790 1791 CSEMap.InsertNode(N, IP); 1792 InsertNode(N); 1793 return SDValue(N, 0); 1794 } 1795 1796 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, 1797 int64_t Offset, 1798 bool isTarget, 1799 unsigned char TargetFlags) { 1800 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; 1801 1802 FoldingSetNodeID ID; 1803 AddNodeIDNode(ID, Opc, getVTList(VT), None); 1804 ID.AddPointer(BA); 1805 ID.AddInteger(Offset); 1806 ID.AddInteger(TargetFlags); 1807 void *IP = nullptr; 1808 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1809 return SDValue(E, 0); 1810 1811 auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags); 1812 CSEMap.InsertNode(N, IP); 1813 InsertNode(N); 1814 return SDValue(N, 0); 1815 } 1816 1817 SDValue SelectionDAG::getSrcValue(const Value *V) { 1818 assert((!V || V->getType()->isPointerTy()) && 1819 "SrcValue is not a pointer?"); 1820 1821 FoldingSetNodeID ID; 1822 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None); 1823 ID.AddPointer(V); 1824 1825 void *IP = nullptr; 1826 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1827 return SDValue(E, 0); 1828 1829 auto *N = newSDNode<SrcValueSDNode>(V); 1830 CSEMap.InsertNode(N, IP); 1831 InsertNode(N); 1832 return SDValue(N, 0); 1833 } 1834 1835 SDValue SelectionDAG::getMDNode(const MDNode *MD) { 1836 FoldingSetNodeID ID; 1837 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None); 1838 ID.AddPointer(MD); 1839 1840 void *IP = nullptr; 1841 if (SDNode *E = FindNodeOrInsertPos(ID, IP)) 1842 return SDValue(E, 0); 1843 1844 auto *N = newSDNode<MDNodeSDNode>(MD); 1845 CSEMap.InsertNode(N, IP); 1846 InsertNode(N); 1847 return SDValue(N, 0); 1848 } 1849 1850 SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) { 1851 if (VT == V.getValueType()) 1852 return V; 1853 1854 return getNode(ISD::BITCAST, SDLoc(V), VT, V); 1855 } 1856 1857 SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, 1858 unsigned SrcAS, unsigned DestAS) { 1859 SDValue Ops[] = {Ptr}; 1860 FoldingSetNodeID ID; 1861 AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops); 1862 ID.AddInteger(SrcAS); 1863 ID.AddInteger(DestAS); 1864 1865 void *IP = nullptr; 1866 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) 1867 return SDValue(E, 0); 1868 1869 auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(), 1870 VT, SrcAS, DestAS); 1871 createOperands(N, Ops); 1872 1873 CSEMap.InsertNode(N, IP); 1874 InsertNode(N); 1875 return SDValue(N, 0); 1876 } 1877 1878 /// getShiftAmountOperand - Return the specified value casted to 1879 /// the target's desired shift amount type. 1880 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { 1881 EVT OpTy = Op.getValueType(); 1882 EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout()); 1883 if (OpTy == ShTy || OpTy.isVector()) return Op; 1884 1885 return getZExtOrTrunc(Op, SDLoc(Op), ShTy); 1886 } 1887 1888 SDValue SelectionDAG::expandVAArg(SDNode *Node) { 1889 SDLoc dl(Node); 1890 const TargetLowering &TLI = getTargetLoweringInfo(); 1891 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 1892 EVT VT = Node->getValueType(0); 1893 SDValue Tmp1 = Node->getOperand(0); 1894 SDValue Tmp2 = Node->getOperand(1); 1895 unsigned Align = Node->getConstantOperandVal(3); 1896 1897 SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1, 1898 Tmp2, MachinePointerInfo(V)); 1899 SDValue VAList = VAListLoad; 1900 1901 if (Align > TLI.getMinStackArgumentAlignment()) { 1902 assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2"); 1903 1904 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, 1905 getConstant(Align - 1, dl, VAList.getValueType())); 1906 1907 VAList = getNode(ISD::AND, dl, VAList.getValueType(), VAList, 1908 getConstant(-(int64_t)Align, dl, VAList.getValueType())); 1909 } 1910 1911 // Increment the pointer, VAList, to the next vaarg 1912 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, 1913 getConstant(getDataLayout().getTypeAllocSize( 1914 VT.getTypeForEVT(*getContext())), 1915 dl, VAList.getValueType())); 1916 // Store the incremented VAList to the legalized pointer 1917 Tmp1 = 1918 getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V)); 1919 // Load the actual argument out of the pointer VAList 1920 return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo()); 1921 } 1922 1923 SDValue SelectionDAG::expandVACopy(SDNode *Node) { 1924 SDLoc dl(Node); 1925 const TargetLowering &TLI = getTargetLoweringInfo(); 1926 // This defaults to loading a pointer from the input and storing it to the 1927 // output, returning the chain. 1928 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue(); 1929 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue(); 1930 SDValue Tmp1 = 1931 getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0), 1932 Node->getOperand(2), MachinePointerInfo(VS)); 1933 return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1), 1934 MachinePointerInfo(VD)); 1935 } 1936 1937 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { 1938 MachineFrameInfo &MFI = getMachineFunction().getFrameInfo(); 1939 unsigned ByteSize = VT.getStoreSize(); 1940 Type *Ty = VT.getTypeForEVT(*getContext()); 1941 unsigned StackAlign = 1942 std::max((unsigned)getDataLayout().getPrefTypeAlignment(Ty), minAlign); 1943 1944 int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false); 1945 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout())); 1946 } 1947 1948 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { 1949 unsigned Bytes = std::max(VT1.getStoreSize(), VT2.getStoreSize()); 1950 Type *Ty1 = VT1.getTypeForEVT(*getContext()); 1951 Type *Ty2 = VT2.getTypeForEVT(*getContext()); 1952 const DataLayout &DL = getDataLayout(); 1953 unsigned Align = 1954 std::max(DL.getPrefTypeAlignment(Ty1), DL.getPrefTypeAlignment(Ty2)); 1955 1956 MachineFrameInfo &MFI = getMachineFunction().getFrameInfo(); 1957 int FrameIdx = MFI.CreateStackObject(Bytes, Align, false); 1958 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout())); 1959 } 1960 1961 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2, 1962 ISD::CondCode Cond, const SDLoc &dl) { 1963 EVT OpVT = N1.getValueType(); 1964 1965 // These setcc operations always fold. 1966 switch (Cond) { 1967 default: break; 1968 case ISD::SETFALSE: 1969 case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT); 1970 case ISD::SETTRUE: 1971 case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT); 1972 1973 case ISD::SETOEQ: 1974 case ISD::SETOGT: 1975 case ISD::SETOGE: 1976 case ISD::SETOLT: 1977 case ISD::SETOLE: 1978 case ISD::SETONE: 1979 case ISD::SETO: 1980 case ISD::SETUO: 1981 case ISD::SETUEQ: 1982 case ISD::SETUNE: 1983 assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!"); 1984 break; 1985 } 1986 1987 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) { 1988 const APInt &C2 = N2C->getAPIntValue(); 1989 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) { 1990 const APInt &C1 = N1C->getAPIntValue(); 1991 1992 switch (Cond) { 1993 default: llvm_unreachable("Unknown integer setcc!"); 1994 case ISD::SETEQ: return getBoolConstant(C1 == C2, dl, VT, OpVT); 1995 case ISD::SETNE: return getBoolConstant(C1 != C2, dl, VT, OpVT); 1996 case ISD::SETULT: return getBoolConstant(C1.ult(C2), dl, VT, OpVT); 1997 case ISD::SETUGT: return getBoolConstant(C1.ugt(C2), dl, VT, OpVT); 1998 case ISD::SETULE: return getBoolConstant(C1.ule(C2), dl, VT, OpVT); 1999 case ISD::SETUGE: return getBoolConstant(C1.uge(C2), dl, VT, OpVT); 2000 case ISD::SETLT: return getBoolConstant(C1.slt(C2), dl, VT, OpVT); 2001 case ISD::SETGT: return getBoolConstant(C1.sgt(C2), dl, VT, OpVT); 2002 case ISD::SETLE: return getBoolConstant(C1.sle(C2), dl, VT, OpVT); 2003 case ISD::SETGE: return getBoolConstant(C1.sge(C2), dl, VT, OpVT); 2004 } 2005 } 2006 } 2007 if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1)) { 2008 if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2)) { 2009 APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF()); 2010 switch (Cond) { 2011 default: break; 2012 case ISD::SETEQ: if (R==APFloat::cmpUnordered) 2013 return getUNDEF(VT); 2014 LLVM_FALLTHROUGH; 2015 case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT, 2016 OpVT); 2017 case ISD::SETNE: if (R==APFloat::cmpUnordered) 2018 return getUNDEF(VT); 2019 LLVM_FALLTHROUGH; 2020 case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan || 2021 R==APFloat::cmpLessThan, dl, VT, 2022 OpVT); 2023 case ISD::SETLT: if (R==APFloat::cmpUnordered) 2024 return getUNDEF(VT); 2025 LLVM_FALLTHROUGH; 2026 case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT, 2027 OpVT); 2028 case ISD::SETGT: if (R==APFloat::cmpUnordered) 2029 return getUNDEF(VT); 2030 LLVM_FALLTHROUGH; 2031 case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl, 2032 VT, OpVT); 2033 case ISD::SETLE: if (R==APFloat::cmpUnordered) 2034 return getUNDEF(VT); 2035 LLVM_FALLTHROUGH; 2036 case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan || 2037 R==APFloat::cmpEqual, dl, VT, 2038 OpVT); 2039 case ISD::SETGE: if (R==APFloat::cmpUnordered) 2040 return getUNDEF(VT); 2041 LLVM_FALLTHROUGH; 2042 case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan || 2043 R==APFloat::cmpEqual, dl, VT, OpVT); 2044 case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT, 2045 OpVT); 2046 case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT, 2047 OpVT); 2048 case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered || 2049 R==APFloat::cmpEqual, dl, VT, 2050 OpVT); 2051 case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT, 2052 OpVT); 2053 case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered || 2054 R==APFloat::cmpLessThan, dl, VT, 2055 OpVT); 2056 case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan || 2057 R==APFloat::cmpUnordered, dl, VT, 2058 OpVT); 2059 case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl, 2060 VT, OpVT); 2061 case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT, 2062 OpVT); 2063 } 2064 } else { 2065 // Ensure that the constant occurs on the RHS. 2066 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond); 2067 MVT CompVT = N1.getValueType().getSimpleVT(); 2068 if (!TLI->isCondCodeLegal(SwappedCond, CompVT)) 2069 return SDValue(); 2070 2071 return getSetCC(dl, VT, N2, N1, SwappedCond); 2072 } 2073 } 2074 2075 // Could not fold it. 2076 return SDValue(); 2077 } 2078 2079 /// See if the specified operand can be simplified with the knowledge that only 2080 /// the bits specified by Mask are used. 2081 SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &Mask) { 2082 switch (V.getOpcode()) { 2083 default: 2084 break; 2085 case ISD::Constant: { 2086 const ConstantSDNode *CV = cast<ConstantSDNode>(V.getNode()); 2087 assert(CV && "Const value should be ConstSDNode."); 2088 const APInt &CVal = CV->getAPIntValue(); 2089 APInt NewVal = CVal & Mask; 2090 if (NewVal != CVal) 2091 return getConstant(NewVal, SDLoc(V), V.getValueType()); 2092 break; 2093 } 2094 case ISD::OR: 2095 case ISD::XOR: 2096 // If the LHS or RHS don't contribute bits to the or, drop them. 2097 if (MaskedValueIsZero(V.getOperand(0), Mask)) 2098 return V.getOperand(1); 2099 if (MaskedValueIsZero(V.getOperand(1), Mask)) 2100 return V.getOperand(0); 2101 break; 2102 case ISD::SRL: 2103 // Only look at single-use SRLs. 2104 if (!V.getNode()->hasOneUse()) 2105 break; 2106 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) { 2107 // See if we can recursively simplify the LHS. 2108 unsigned Amt = RHSC->getZExtValue(); 2109 2110 // Watch out for shift count overflow though. 2111 if (Amt >= Mask.getBitWidth()) 2112 break; 2113 APInt NewMask = Mask << Amt; 2114 if (SDValue SimplifyLHS = GetDemandedBits(V.getOperand(0), NewMask)) 2115 return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS, 2116 V.getOperand(1)); 2117 } 2118 break; 2119 case ISD::AND: { 2120 // X & -1 -> X (ignoring bits which aren't demanded). 2121 ConstantSDNode *AndVal = isConstOrConstSplat(V.getOperand(1)); 2122 if (AndVal && Mask.isSubsetOf(AndVal->getAPIntValue())) 2123 return V.getOperand(0); 2124 break; 2125 } 2126 case ISD::ANY_EXTEND: { 2127 SDValue Src = V.getOperand(0); 2128 unsigned SrcBitWidth = Src.getScalarValueSizeInBits(); 2129 // Being conservative here - only peek through if we only demand bits in the 2130 // non-extended source (even though the extended bits are technically undef). 2131 if (Mask.getActiveBits() > SrcBitWidth) 2132 break; 2133 APInt SrcMask = Mask.trunc(SrcBitWidth); 2134 if (SDValue DemandedSrc = GetDemandedBits(Src, SrcMask)) 2135 return getNode(ISD::ANY_EXTEND, SDLoc(V), V.getValueType(), DemandedSrc); 2136 break; 2137 } 2138 } 2139 return SDValue(); 2140 } 2141 2142 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We 2143 /// use this predicate to simplify operations downstream. 2144 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { 2145 unsigned BitWidth = Op.getScalarValueSizeInBits(); 2146 return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth); 2147 } 2148 2149 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use 2150 /// this predicate to simplify operations downstream. Mask is known to be zero 2151 /// for bits that V cannot have. 2152 bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask, 2153 unsigned Depth) const { 2154 KnownBits Known; 2155 computeKnownBits(Op, Known, Depth); 2156 return Mask.isSubsetOf(Known.Zero); 2157 } 2158 2159 /// Helper function that checks to see if a node is a constant or a 2160 /// build vector of splat constants at least within the demanded elts. 2161 static ConstantSDNode *isConstOrDemandedConstSplat(SDValue N, 2162 const APInt &DemandedElts) { 2163 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) 2164 return CN; 2165 if (N.getOpcode() != ISD::BUILD_VECTOR) 2166 return nullptr; 2167 EVT VT = N.getValueType(); 2168 ConstantSDNode *Cst = nullptr; 2169 unsigned NumElts = VT.getVectorNumElements(); 2170 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected vector size"); 2171 for (unsigned i = 0; i != NumElts; ++i) { 2172 if (!DemandedElts[i]) 2173 continue; 2174 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(i)); 2175 if (!C || (Cst && Cst->getAPIntValue() != C->getAPIntValue()) || 2176 C->getValueType(0) != VT.getScalarType()) 2177 return nullptr; 2178 Cst = C; 2179 } 2180 return Cst; 2181 } 2182 2183 /// If a SHL/SRA/SRL node has a constant or splat constant shift amount that 2184 /// is less than the element bit-width of the shift node, return it. 2185 static const APInt *getValidShiftAmountConstant(SDValue V) { 2186 if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1))) { 2187 // Shifting more than the bitwidth is not valid. 2188 const APInt &ShAmt = SA->getAPIntValue(); 2189 if (ShAmt.ult(V.getScalarValueSizeInBits())) 2190 return &ShAmt; 2191 } 2192 return nullptr; 2193 } 2194 2195 /// Determine which bits of Op are known to be either zero or one and return 2196 /// them in Known. For vectors, the known bits are those that are shared by 2197 /// every vector element. 2198 void SelectionDAG::computeKnownBits(SDValue Op, KnownBits &Known, 2199 unsigned Depth) const { 2200 EVT VT = Op.getValueType(); 2201 APInt DemandedElts = VT.isVector() 2202 ? APInt::getAllOnesValue(VT.getVectorNumElements()) 2203 : APInt(1, 1); 2204 computeKnownBits(Op, Known, DemandedElts, Depth); 2205 } 2206 2207 /// Determine which bits of Op are known to be either zero or one and return 2208 /// them in Known. The DemandedElts argument allows us to only collect the known 2209 /// bits that are shared by the requested vector elements. 2210 void SelectionDAG::computeKnownBits(SDValue Op, KnownBits &Known, 2211 const APInt &DemandedElts, 2212 unsigned Depth) const { 2213 unsigned BitWidth = Op.getScalarValueSizeInBits(); 2214 2215 Known = KnownBits(BitWidth); // Don't know anything. 2216 2217 if (auto *C = dyn_cast<ConstantSDNode>(Op)) { 2218 // We know all of the bits for a constant! 2219 Known.One = C->getAPIntValue(); 2220 Known.Zero = ~Known.One; 2221 return; 2222 } 2223 if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) { 2224 // We know all of the bits for a constant fp! 2225 Known.One = C->getValueAPF().bitcastToAPInt(); 2226 Known.Zero = ~Known.One; 2227 return; 2228 } 2229 2230 if (Depth == 6) 2231 return; // Limit search depth. 2232 2233 KnownBits Known2; 2234 unsigned NumElts = DemandedElts.getBitWidth(); 2235 2236 if (!DemandedElts) 2237 return; // No demanded elts, better to assume we don't know anything. 2238 2239 unsigned Opcode = Op.getOpcode(); 2240 switch (Opcode) { 2241 case ISD::BUILD_VECTOR: 2242 // Collect the known bits that are shared by every demanded vector element. 2243 assert(NumElts == Op.getValueType().getVectorNumElements() && 2244 "Unexpected vector size"); 2245 Known.Zero.setAllBits(); Known.One.setAllBits(); 2246 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) { 2247 if (!DemandedElts[i]) 2248 continue; 2249 2250 SDValue SrcOp = Op.getOperand(i); 2251 computeKnownBits(SrcOp, Known2, Depth + 1); 2252 2253 // BUILD_VECTOR can implicitly truncate sources, we must handle this. 2254 if (SrcOp.getValueSizeInBits() != BitWidth) { 2255 assert(SrcOp.getValueSizeInBits() > BitWidth && 2256 "Expected BUILD_VECTOR implicit truncation"); 2257 Known2 = Known2.trunc(BitWidth); 2258 } 2259 2260 // Known bits are the values that are shared by every demanded element. 2261 Known.One &= Known2.One; 2262 Known.Zero &= Known2.Zero; 2263 2264 // If we don't know any bits, early out. 2265 if (Known.isUnknown()) 2266 break; 2267 } 2268 break; 2269 case ISD::VECTOR_SHUFFLE: { 2270 // Collect the known bits that are shared by every vector element referenced 2271 // by the shuffle. 2272 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0); 2273 Known.Zero.setAllBits(); Known.One.setAllBits(); 2274 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op); 2275 assert(NumElts == SVN->getMask().size() && "Unexpected vector size"); 2276 for (unsigned i = 0; i != NumElts; ++i) { 2277 if (!DemandedElts[i]) 2278 continue; 2279 2280 int M = SVN->getMaskElt(i); 2281 if (M < 0) { 2282 // For UNDEF elements, we don't know anything about the common state of 2283 // the shuffle result. 2284 Known.resetAll(); 2285 DemandedLHS.clearAllBits(); 2286 DemandedRHS.clearAllBits(); 2287 break; 2288 } 2289 2290 if ((unsigned)M < NumElts) 2291 DemandedLHS.setBit((unsigned)M % NumElts); 2292 else 2293 DemandedRHS.setBit((unsigned)M % NumElts); 2294 } 2295 // Known bits are the values that are shared by every demanded element. 2296 if (!!DemandedLHS) { 2297 SDValue LHS = Op.getOperand(0); 2298 computeKnownBits(LHS, Known2, DemandedLHS, Depth + 1); 2299 Known.One &= Known2.One; 2300 Known.Zero &= Known2.Zero; 2301 } 2302 // If we don't know any bits, early out. 2303 if (Known.isUnknown()) 2304 break; 2305 if (!!DemandedRHS) { 2306 SDValue RHS = Op.getOperand(1); 2307 computeKnownBits(RHS, Known2, DemandedRHS, Depth + 1); 2308 Known.One &= Known2.One; 2309 Known.Zero &= Known2.Zero; 2310 } 2311 break; 2312 } 2313 case ISD::CONCAT_VECTORS: { 2314 // Split DemandedElts and test each of the demanded subvectors. 2315 Known.Zero.setAllBits(); Known.One.setAllBits(); 2316 EVT SubVectorVT = Op.getOperand(0).getValueType(); 2317 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements(); 2318 unsigned NumSubVectors = Op.getNumOperands(); 2319 for (unsigned i = 0; i != NumSubVectors; ++i) { 2320 APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts); 2321 DemandedSub = DemandedSub.trunc(NumSubVectorElts); 2322 if (!!DemandedSub) { 2323 SDValue Sub = Op.getOperand(i); 2324 computeKnownBits(Sub, Known2, DemandedSub, Depth + 1); 2325 Known.One &= Known2.One; 2326 Known.Zero &= Known2.Zero; 2327 } 2328 // If we don't know any bits, early out. 2329 if (Known.isUnknown()) 2330 break; 2331 } 2332 break; 2333 } 2334 case ISD::INSERT_SUBVECTOR: { 2335 // If we know the element index, demand any elements from the subvector and 2336 // the remainder from the src its inserted into, otherwise demand them all. 2337 SDValue Src = Op.getOperand(0); 2338 SDValue Sub = Op.getOperand(1); 2339 ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2)); 2340 unsigned NumSubElts = Sub.getValueType().getVectorNumElements(); 2341 if (SubIdx && SubIdx->getAPIntValue().ule(NumElts - NumSubElts)) { 2342 Known.One.setAllBits(); 2343 Known.Zero.setAllBits(); 2344 uint64_t Idx = SubIdx->getZExtValue(); 2345 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx); 2346 if (!!DemandedSubElts) { 2347 computeKnownBits(Sub, Known, DemandedSubElts, Depth + 1); 2348 if (Known.isUnknown()) 2349 break; // early-out. 2350 } 2351 APInt SubMask = APInt::getBitsSet(NumElts, Idx, Idx + NumSubElts); 2352 APInt DemandedSrcElts = DemandedElts & ~SubMask; 2353 if (!!DemandedSrcElts) { 2354 computeKnownBits(Src, Known2, DemandedSrcElts, Depth + 1); 2355 Known.One &= Known2.One; 2356 Known.Zero &= Known2.Zero; 2357 } 2358 } else { 2359 computeKnownBits(Sub, Known, Depth + 1); 2360 if (Known.isUnknown()) 2361 break; // early-out. 2362 computeKnownBits(Src, Known2, Depth + 1); 2363 Known.One &= Known2.One; 2364 Known.Zero &= Known2.Zero; 2365 } 2366 break; 2367 } 2368 case ISD::EXTRACT_SUBVECTOR: { 2369 // If we know the element index, just demand that subvector elements, 2370 // otherwise demand them all. 2371 SDValue Src = Op.getOperand(0); 2372 ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 2373 unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); 2374 if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) { 2375 // Offset the demanded elts by the subvector index. 2376 uint64_t Idx = SubIdx->getZExtValue(); 2377 APInt DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx); 2378 computeKnownBits(Src, Known, DemandedSrc, Depth + 1); 2379 } else { 2380 computeKnownBits(Src, Known, Depth + 1); 2381 } 2382 break; 2383 } 2384 case ISD::BITCAST: { 2385 SDValue N0 = Op.getOperand(0); 2386 EVT SubVT = N0.getValueType(); 2387 unsigned SubBitWidth = SubVT.getScalarSizeInBits(); 2388 2389 // Ignore bitcasts from unsupported types. 2390 if (!(SubVT.isInteger() || SubVT.isFloatingPoint())) 2391 break; 2392 2393 // Fast handling of 'identity' bitcasts. 2394 if (BitWidth == SubBitWidth) { 2395 computeKnownBits(N0, Known, DemandedElts, Depth + 1); 2396 break; 2397 } 2398 2399 bool IsLE = getDataLayout().isLittleEndian(); 2400 2401 // Bitcast 'small element' vector to 'large element' scalar/vector. 2402 if ((BitWidth % SubBitWidth) == 0) { 2403 assert(N0.getValueType().isVector() && "Expected bitcast from vector"); 2404 2405 // Collect known bits for the (larger) output by collecting the known 2406 // bits from each set of sub elements and shift these into place. 2407 // We need to separately call computeKnownBits for each set of 2408 // sub elements as the knownbits for each is likely to be different. 2409 unsigned SubScale = BitWidth / SubBitWidth; 2410 APInt SubDemandedElts(NumElts * SubScale, 0); 2411 for (unsigned i = 0; i != NumElts; ++i) 2412 if (DemandedElts[i]) 2413 SubDemandedElts.setBit(i * SubScale); 2414 2415 for (unsigned i = 0; i != SubScale; ++i) { 2416 computeKnownBits(N0, Known2, SubDemandedElts.shl(i), 2417 Depth + 1); 2418 unsigned Shifts = IsLE ? i : SubScale - 1 - i; 2419 Known.One |= Known2.One.zext(BitWidth).shl(SubBitWidth * Shifts); 2420 Known.Zero |= Known2.Zero.zext(BitWidth).shl(SubBitWidth * Shifts); 2421 } 2422 } 2423 2424 // Bitcast 'large element' scalar/vector to 'small element' vector. 2425 if ((SubBitWidth % BitWidth) == 0) { 2426 assert(Op.getValueType().isVector() && "Expected bitcast to vector"); 2427 2428 // Collect known bits for the (smaller) output by collecting the known 2429 // bits from the overlapping larger input elements and extracting the 2430 // sub sections we actually care about. 2431 unsigned SubScale = SubBitWidth / BitWidth; 2432 APInt SubDemandedElts(NumElts / SubScale, 0); 2433 for (unsigned i = 0; i != NumElts; ++i) 2434 if (DemandedElts[i]) 2435 SubDemandedElts.setBit(i / SubScale); 2436 2437 computeKnownBits(N0, Known2, SubDemandedElts, Depth + 1); 2438 2439 Known.Zero.setAllBits(); Known.One.setAllBits(); 2440 for (unsigned i = 0; i != NumElts; ++i) 2441 if (DemandedElts[i]) { 2442 unsigned Shifts = IsLE ? i : NumElts - 1 - i; 2443 unsigned Offset = (Shifts % SubScale) * BitWidth; 2444 Known.One &= Known2.One.lshr(Offset).trunc(BitWidth); 2445 Known.Zero &= Known2.Zero.lshr(Offset).trunc(BitWidth); 2446 // If we don't know any bits, early out. 2447 if (Known.isUnknown()) 2448 break; 2449 } 2450 } 2451 break; 2452 } 2453 case ISD::AND: 2454 // If either the LHS or the RHS are Zero, the result is zero. 2455 computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1); 2456 computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); 2457 2458 // Output known-1 bits are only known if set in both the LHS & RHS. 2459 Known.One &= Known2.One; 2460 // Output known-0 are known to be clear if zero in either the LHS | RHS. 2461 Known.Zero |= Known2.Zero; 2462 break; 2463 case ISD::OR: 2464 computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1); 2465 computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); 2466 2467 // Output known-0 bits are only known if clear in both the LHS & RHS. 2468 Known.Zero &= Known2.Zero; 2469 // Output known-1 are known to be set if set in either the LHS | RHS. 2470 Known.One |= Known2.One; 2471 break; 2472 case ISD::XOR: { 2473 computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1); 2474 computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); 2475 2476 // Output known-0 bits are known if clear or set in both the LHS & RHS. 2477 APInt KnownZeroOut = (Known.Zero & Known2.Zero) | (Known.One & Known2.One); 2478 // Output known-1 are known to be set if set in only one of the LHS, RHS. 2479 Known.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero); 2480 Known.Zero = KnownZeroOut; 2481 break; 2482 } 2483 case ISD::MUL: { 2484 computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1); 2485 computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); 2486 2487 // If low bits are zero in either operand, output low known-0 bits. 2488 // Also compute a conservative estimate for high known-0 bits. 2489 // More trickiness is possible, but this is sufficient for the 2490 // interesting case of alignment computation. 2491 unsigned TrailZ = Known.countMinTrailingZeros() + 2492 Known2.countMinTrailingZeros(); 2493 unsigned LeadZ = std::max(Known.countMinLeadingZeros() + 2494 Known2.countMinLeadingZeros(), 2495 BitWidth) - BitWidth; 2496 2497 Known.resetAll(); 2498 Known.Zero.setLowBits(std::min(TrailZ, BitWidth)); 2499 Known.Zero.setHighBits(std::min(LeadZ, BitWidth)); 2500 break; 2501 } 2502 case ISD::UDIV: { 2503 // For the purposes of computing leading zeros we can conservatively 2504 // treat a udiv as a logical right shift by the power of 2 known to 2505 // be less than the denominator. 2506 computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); 2507 unsigned LeadZ = Known2.countMinLeadingZeros(); 2508 2509 computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1); 2510 unsigned RHSMaxLeadingZeros = Known2.countMaxLeadingZeros(); 2511 if (RHSMaxLeadingZeros != BitWidth) 2512 LeadZ = std::min(BitWidth, LeadZ + BitWidth - RHSMaxLeadingZeros - 1); 2513 2514 Known.Zero.setHighBits(LeadZ); 2515 break; 2516 } 2517 case ISD::SELECT: 2518 case ISD::VSELECT: 2519 computeKnownBits(Op.getOperand(2), Known, DemandedElts, Depth+1); 2520 // If we don't know any bits, early out. 2521 if (Known.isUnknown()) 2522 break; 2523 computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth+1); 2524 2525 // Only known if known in both the LHS and RHS. 2526 Known.One &= Known2.One; 2527 Known.Zero &= Known2.Zero; 2528 break; 2529 case ISD::SELECT_CC: 2530 computeKnownBits(Op.getOperand(3), Known, DemandedElts, Depth+1); 2531 // If we don't know any bits, early out. 2532 if (Known.isUnknown()) 2533 break; 2534 computeKnownBits(Op.getOperand(2), Known2, DemandedElts, Depth+1); 2535 2536 // Only known if known in both the LHS and RHS. 2537 Known.One &= Known2.One; 2538 Known.Zero &= Known2.Zero; 2539 break; 2540 case ISD::SMULO: 2541 case ISD::UMULO: 2542 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 2543 if (Op.getResNo() != 1) 2544 break; 2545 // The boolean result conforms to getBooleanContents. 2546 // If we know the result of a setcc has the top bits zero, use this info. 2547 // We know that we have an integer-based boolean since these operations 2548 // are only available for integer. 2549 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) == 2550 TargetLowering::ZeroOrOneBooleanContent && 2551 BitWidth > 1) 2552 Known.Zero.setBitsFrom(1); 2553 break; 2554 case ISD::SETCC: 2555 // If we know the result of a setcc has the top bits zero, use this info. 2556 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == 2557 TargetLowering::ZeroOrOneBooleanContent && 2558 BitWidth > 1) 2559 Known.Zero.setBitsFrom(1); 2560 break; 2561 case ISD::SHL: 2562 if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) { 2563 computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); 2564 unsigned Shift = ShAmt->getZExtValue(); 2565 Known.Zero <<= Shift; 2566 Known.One <<= Shift; 2567 // Low bits are known zero. 2568 Known.Zero.setLowBits(Shift); 2569 } 2570 break; 2571 case ISD::SRL: 2572 if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) { 2573 computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); 2574 unsigned Shift = ShAmt->getZExtValue(); 2575 Known.Zero.lshrInPlace(Shift); 2576 Known.One.lshrInPlace(Shift); 2577 // High bits are known zero. 2578 Known.Zero.setHighBits(Shift); 2579 } else if (auto *BV = dyn_cast<BuildVectorSDNode>(Op.getOperand(1))) { 2580 // If the shift amount is a vector of constants see if we can bound 2581 // the number of upper zero bits. 2582 unsigned ShiftAmountMin = BitWidth; 2583 for (unsigned i = 0; i != BV->getNumOperands(); ++i) { 2584 if (auto *C = dyn_cast<ConstantSDNode>(BV->getOperand(i))) { 2585 const APInt &ShAmt = C->getAPIntValue(); 2586 if (ShAmt.ult(BitWidth)) { 2587 ShiftAmountMin = std::min<unsigned>(ShiftAmountMin, 2588 ShAmt.getZExtValue()); 2589 continue; 2590 } 2591 } 2592 // Don't know anything. 2593 ShiftAmountMin = 0; 2594 break; 2595 } 2596 2597 Known.Zero.setHighBits(ShiftAmountMin); 2598 } 2599 break; 2600 case ISD::SRA: 2601 if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) { 2602 computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); 2603 unsigned Shift = ShAmt->getZExtValue(); 2604 // Sign extend known zero/one bit (else is unknown). 2605 Known.Zero.ashrInPlace(Shift); 2606 Known.One.ashrInPlace(Shift); 2607 } 2608 break; 2609 case ISD::SIGN_EXTEND_INREG: { 2610 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 2611 unsigned EBits = EVT.getScalarSizeInBits(); 2612 2613 // Sign extension. Compute the demanded bits in the result that are not 2614 // present in the input. 2615 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits); 2616 2617 APInt InSignMask = APInt::getSignMask(EBits); 2618 APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits); 2619 2620 // If the sign extended bits are demanded, we know that the sign 2621 // bit is demanded. 2622 InSignMask = InSignMask.zext(BitWidth); 2623 if (NewBits.getBoolValue()) 2624 InputDemandedBits |= InSignMask; 2625 2626 computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); 2627 Known.One &= InputDemandedBits; 2628 Known.Zero &= InputDemandedBits; 2629 2630 // If the sign bit of the input is known set or clear, then we know the 2631 // top bits of the result. 2632 if (Known.Zero.intersects(InSignMask)) { // Input sign bit known clear 2633 Known.Zero |= NewBits; 2634 Known.One &= ~NewBits; 2635 } else if (Known.One.intersects(InSignMask)) { // Input sign bit known set 2636 Known.One |= NewBits; 2637 Known.Zero &= ~NewBits; 2638 } else { // Input sign bit unknown 2639 Known.Zero &= ~NewBits; 2640 Known.One &= ~NewBits; 2641 } 2642 break; 2643 } 2644 case ISD::CTTZ: 2645 case ISD::CTTZ_ZERO_UNDEF: { 2646 computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); 2647 // If we have a known 1, its position is our upper bound. 2648 unsigned PossibleTZ = Known2.countMaxTrailingZeros(); 2649 unsigned LowBits = Log2_32(PossibleTZ) + 1; 2650 Known.Zero.setBitsFrom(LowBits); 2651 break; 2652 } 2653 case ISD::CTLZ: 2654 case ISD::CTLZ_ZERO_UNDEF: { 2655 computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); 2656 // If we have a known 1, its position is our upper bound. 2657 unsigned PossibleLZ = Known2.countMaxLeadingZeros(); 2658 unsigned LowBits = Log2_32(PossibleLZ) + 1; 2659 Known.Zero.setBitsFrom(LowBits); 2660 break; 2661 } 2662 case ISD::CTPOP: { 2663 computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); 2664 // If we know some of the bits are zero, they can't be one. 2665 unsigned PossibleOnes = Known2.countMaxPopulation(); 2666 Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1); 2667 break; 2668 } 2669 case ISD::LOAD: { 2670 LoadSDNode *LD = cast<LoadSDNode>(Op); 2671 // If this is a ZEXTLoad and we are looking at the loaded value. 2672 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { 2673 EVT VT = LD->getMemoryVT(); 2674 unsigned MemBits = VT.getScalarSizeInBits(); 2675 Known.Zero.setBitsFrom(MemBits); 2676 } else if (const MDNode *Ranges = LD->getRanges()) { 2677 if (LD->getExtensionType() == ISD::NON_EXTLOAD) 2678 computeKnownBitsFromRangeMetadata(*Ranges, Known); 2679 } 2680 break; 2681 } 2682 case ISD::ZERO_EXTEND_VECTOR_INREG: { 2683 EVT InVT = Op.getOperand(0).getValueType(); 2684 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements()); 2685 computeKnownBits(Op.getOperand(0), Known, InDemandedElts, Depth + 1); 2686 Known = Known.zext(BitWidth); 2687 Known.Zero.setBitsFrom(InVT.getScalarSizeInBits()); 2688 break; 2689 } 2690 case ISD::ZERO_EXTEND: { 2691 EVT InVT = Op.getOperand(0).getValueType(); 2692 computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); 2693 Known = Known.zext(BitWidth); 2694 Known.Zero.setBitsFrom(InVT.getScalarSizeInBits()); 2695 break; 2696 } 2697 // TODO ISD::SIGN_EXTEND_VECTOR_INREG 2698 case ISD::SIGN_EXTEND: { 2699 computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); 2700 // If the sign bit is known to be zero or one, then sext will extend 2701 // it to the top bits, else it will just zext. 2702 Known = Known.sext(BitWidth); 2703 break; 2704 } 2705 case ISD::ANY_EXTEND: { 2706 computeKnownBits(Op.getOperand(0), Known, Depth+1); 2707 Known = Known.zext(BitWidth); 2708 break; 2709 } 2710 case ISD::TRUNCATE: { 2711 computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); 2712 Known = Known.trunc(BitWidth); 2713 break; 2714 } 2715 case ISD::AssertZext: { 2716 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 2717 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); 2718 computeKnownBits(Op.getOperand(0), Known, Depth+1); 2719 Known.Zero |= (~InMask); 2720 Known.One &= (~Known.Zero); 2721 break; 2722 } 2723 case ISD::FGETSIGN: 2724 // All bits are zero except the low bit. 2725 Known.Zero.setBitsFrom(1); 2726 break; 2727 case ISD::USUBO: 2728 case ISD::SSUBO: 2729 if (Op.getResNo() == 1) { 2730 // If we know the result of a setcc has the top bits zero, use this info. 2731 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == 2732 TargetLowering::ZeroOrOneBooleanContent && 2733 BitWidth > 1) 2734 Known.Zero.setBitsFrom(1); 2735 break; 2736 } 2737 LLVM_FALLTHROUGH; 2738 case ISD::SUB: 2739 case ISD::SUBC: { 2740 if (ConstantSDNode *CLHS = isConstOrConstSplat(Op.getOperand(0))) { 2741 // We know that the top bits of C-X are clear if X contains less bits 2742 // than C (i.e. no wrap-around can happen). For example, 20-X is 2743 // positive if we can prove that X is >= 0 and < 16. 2744 if (CLHS->getAPIntValue().isNonNegative()) { 2745 unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros(); 2746 // NLZ can't be BitWidth with no sign bit 2747 APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1); 2748 computeKnownBits(Op.getOperand(1), Known2, DemandedElts, 2749 Depth + 1); 2750 2751 // If all of the MaskV bits are known to be zero, then we know the 2752 // output top bits are zero, because we now know that the output is 2753 // from [0-C]. 2754 if ((Known2.Zero & MaskV) == MaskV) { 2755 unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros(); 2756 // Top bits known zero. 2757 Known.Zero.setHighBits(NLZ2); 2758 } 2759 } 2760 } 2761 2762 // If low bits are know to be zero in both operands, then we know they are 2763 // going to be 0 in the result. Both addition and complement operations 2764 // preserve the low zero bits. 2765 computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); 2766 unsigned KnownZeroLow = Known2.countMinTrailingZeros(); 2767 if (KnownZeroLow == 0) 2768 break; 2769 2770 computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1); 2771 KnownZeroLow = std::min(KnownZeroLow, Known2.countMinTrailingZeros()); 2772 Known.Zero.setLowBits(KnownZeroLow); 2773 break; 2774 } 2775 case ISD::UADDO: 2776 case ISD::SADDO: 2777 case ISD::ADDCARRY: 2778 if (Op.getResNo() == 1) { 2779 // If we know the result of a setcc has the top bits zero, use this info. 2780 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == 2781 TargetLowering::ZeroOrOneBooleanContent && 2782 BitWidth > 1) 2783 Known.Zero.setBitsFrom(1); 2784 break; 2785 } 2786 LLVM_FALLTHROUGH; 2787 case ISD::ADD: 2788 case ISD::ADDC: 2789 case ISD::ADDE: { 2790 // Output known-0 bits are known if clear or set in both the low clear bits 2791 // common to both LHS & RHS. For example, 8+(X<<3) is known to have the 2792 // low 3 bits clear. 2793 // Output known-0 bits are also known if the top bits of each input are 2794 // known to be clear. For example, if one input has the top 10 bits clear 2795 // and the other has the top 8 bits clear, we know the top 7 bits of the 2796 // output must be clear. 2797 computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); 2798 unsigned KnownZeroHigh = Known2.countMinLeadingZeros(); 2799 unsigned KnownZeroLow = Known2.countMinTrailingZeros(); 2800 2801 computeKnownBits(Op.getOperand(1), Known2, DemandedElts, 2802 Depth + 1); 2803 KnownZeroHigh = std::min(KnownZeroHigh, Known2.countMinLeadingZeros()); 2804 KnownZeroLow = std::min(KnownZeroLow, Known2.countMinTrailingZeros()); 2805 2806 if (Opcode == ISD::ADDE || Opcode == ISD::ADDCARRY) { 2807 // With ADDE and ADDCARRY, a carry bit may be added in, so we can only 2808 // use this information if we know (at least) that the low two bits are 2809 // clear. We then return to the caller that the low bit is unknown but 2810 // that other bits are known zero. 2811 if (KnownZeroLow >= 2) 2812 Known.Zero.setBits(1, KnownZeroLow); 2813 break; 2814 } 2815 2816 Known.Zero.setLowBits(KnownZeroLow); 2817 if (KnownZeroHigh > 1) 2818 Known.Zero.setHighBits(KnownZeroHigh - 1); 2819 break; 2820 } 2821 case ISD::SREM: 2822 if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) { 2823 const APInt &RA = Rem->getAPIntValue().abs(); 2824 if (RA.isPowerOf2()) { 2825 APInt LowBits = RA - 1; 2826 computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); 2827 2828 // The low bits of the first operand are unchanged by the srem. 2829 Known.Zero = Known2.Zero & LowBits; 2830 Known.One = Known2.One & LowBits; 2831 2832 // If the first operand is non-negative or has all low bits zero, then 2833 // the upper bits are all zero. 2834 if (Known2.Zero[BitWidth-1] || ((Known2.Zero & LowBits) == LowBits)) 2835 Known.Zero |= ~LowBits; 2836 2837 // If the first operand is negative and not all low bits are zero, then 2838 // the upper bits are all one. 2839 if (Known2.One[BitWidth-1] && ((Known2.One & LowBits) != 0)) 2840 Known.One |= ~LowBits; 2841 assert((Known.Zero & Known.One) == 0&&"Bits known to be one AND zero?"); 2842 } 2843 } 2844 break; 2845 case ISD::UREM: { 2846 if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) { 2847 const APInt &RA = Rem->getAPIntValue(); 2848 if (RA.isPowerOf2()) { 2849 APInt LowBits = (RA - 1); 2850 computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); 2851 2852 // The upper bits are all zero, the lower ones are unchanged. 2853 Known.Zero = Known2.Zero | ~LowBits; 2854 Known.One = Known2.One & LowBits; 2855 break; 2856 } 2857 } 2858 2859 // Since the result is less than or equal to either operand, any leading 2860 // zero bits in either operand must also exist in the result. 2861 computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); 2862 computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1); 2863 2864 uint32_t Leaders = 2865 std::max(Known.countMinLeadingZeros(), Known2.countMinLeadingZeros()); 2866 Known.resetAll(); 2867 Known.Zero.setHighBits(Leaders); 2868 break; 2869 } 2870 case ISD::EXTRACT_ELEMENT: { 2871 computeKnownBits(Op.getOperand(0), Known, Depth+1); 2872 const unsigned Index = Op.getConstantOperandVal(1); 2873 const unsigned BitWidth = Op.getValueSizeInBits(); 2874 2875 // Remove low part of known bits mask 2876 Known.Zero = Known.Zero.getHiBits(Known.Zero.getBitWidth() - Index * BitWidth); 2877 Known.One = Known.One.getHiBits(Known.One.getBitWidth() - Index * BitWidth); 2878 2879 // Remove high part of known bit mask 2880 Known = Known.trunc(BitWidth); 2881 break; 2882 } 2883 case ISD::EXTRACT_VECTOR_ELT: { 2884 SDValue InVec = Op.getOperand(0); 2885 SDValue EltNo = Op.getOperand(1); 2886 EVT VecVT = InVec.getValueType(); 2887 const unsigned BitWidth = Op.getValueSizeInBits(); 2888 const unsigned EltBitWidth = VecVT.getScalarSizeInBits(); 2889 const unsigned NumSrcElts = VecVT.getVectorNumElements(); 2890 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know 2891 // anything about the extended bits. 2892 if (BitWidth > EltBitWidth) 2893 Known = Known.trunc(EltBitWidth); 2894 ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo); 2895 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)) { 2896 // If we know the element index, just demand that vector element. 2897 unsigned Idx = ConstEltNo->getZExtValue(); 2898 APInt DemandedElt = APInt::getOneBitSet(NumSrcElts, Idx); 2899 computeKnownBits(InVec, Known, DemandedElt, Depth + 1); 2900 } else { 2901 // Unknown element index, so ignore DemandedElts and demand them all. 2902 computeKnownBits(InVec, Known, Depth + 1); 2903 } 2904 if (BitWidth > EltBitWidth) 2905 Known = Known.zext(BitWidth); 2906 break; 2907 } 2908 case ISD::INSERT_VECTOR_ELT: { 2909 SDValue InVec = Op.getOperand(0); 2910 SDValue InVal = Op.getOperand(1); 2911 SDValue EltNo = Op.getOperand(2); 2912 2913 ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo); 2914 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) { 2915 // If we know the element index, split the demand between the 2916 // source vector and the inserted element. 2917 Known.Zero = Known.One = APInt::getAllOnesValue(BitWidth); 2918 unsigned EltIdx = CEltNo->getZExtValue(); 2919 2920 // If we demand the inserted element then add its common known bits. 2921 if (DemandedElts[EltIdx]) { 2922 computeKnownBits(InVal, Known2, Depth + 1); 2923 Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth()); 2924 Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth()); 2925 } 2926 2927 // If we demand the source vector then add its common known bits, ensuring 2928 // that we don't demand the inserted element. 2929 APInt VectorElts = DemandedElts & ~(APInt::getOneBitSet(NumElts, EltIdx)); 2930 if (!!VectorElts) { 2931 computeKnownBits(InVec, Known2, VectorElts, Depth + 1); 2932 Known.One &= Known2.One; 2933 Known.Zero &= Known2.Zero; 2934 } 2935 } else { 2936 // Unknown element index, so ignore DemandedElts and demand them all. 2937 computeKnownBits(InVec, Known, Depth + 1); 2938 computeKnownBits(InVal, Known2, Depth + 1); 2939 Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth()); 2940 Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth()); 2941 } 2942 break; 2943 } 2944 case ISD::BITREVERSE: { 2945 computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); 2946 Known.Zero = Known2.Zero.reverseBits(); 2947 Known.One = Known2.One.reverseBits(); 2948 break; 2949 } 2950 case ISD::BSWAP: { 2951 computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); 2952 Known.Zero = Known2.Zero.byteSwap(); 2953 Known.One = Known2.One.byteSwap(); 2954 break; 2955 } 2956 case ISD::ABS: { 2957 computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); 2958 2959 // If the source's MSB is zero then we know the rest of the bits already. 2960 if (Known2.isNonNegative()) { 2961 Known.Zero = Known2.Zero; 2962 Known.One = Known2.One; 2963 break; 2964 } 2965 2966 // We only know that the absolute values's MSB will be zero iff there is 2967 // a set bit that isn't the sign bit (otherwise it could be INT_MIN). 2968 Known2.One.clearSignBit(); 2969 if (Known2.One.getBoolValue()) { 2970 Known.Zero = APInt::getSignMask(BitWidth); 2971 break; 2972 } 2973 break; 2974 } 2975 case ISD::UMIN: { 2976 computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); 2977 computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1); 2978 2979 // UMIN - we know that the result will have the maximum of the 2980 // known zero leading bits of the inputs. 2981 unsigned LeadZero = Known.countMinLeadingZeros(); 2982 LeadZero = std::max(LeadZero, Known2.countMinLeadingZeros()); 2983 2984 Known.Zero &= Known2.Zero; 2985 Known.One &= Known2.One; 2986 Known.Zero.setHighBits(LeadZero); 2987 break; 2988 } 2989 case ISD::UMAX: { 2990 computeKnownBits(Op.getOperand(0), Known, DemandedElts, 2991 Depth + 1); 2992 computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1); 2993 2994 // UMAX - we know that the result will have the maximum of the 2995 // known one leading bits of the inputs. 2996 unsigned LeadOne = Known.countMinLeadingOnes(); 2997 LeadOne = std::max(LeadOne, Known2.countMinLeadingOnes()); 2998 2999 Known.Zero &= Known2.Zero; 3000 Known.One &= Known2.One; 3001 Known.One.setHighBits(LeadOne); 3002 break; 3003 } 3004 case ISD::SMIN: 3005 case ISD::SMAX: { 3006 // If we have a clamp pattern, we know that the number of sign bits will be 3007 // the minimum of the clamp min/max range. 3008 bool IsMax = (Opcode == ISD::SMAX); 3009 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr; 3010 if ((CstLow = isConstOrDemandedConstSplat(Op.getOperand(1), DemandedElts))) 3011 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) 3012 CstHigh = isConstOrDemandedConstSplat(Op.getOperand(0).getOperand(1), 3013 DemandedElts); 3014 if (CstLow && CstHigh) { 3015 if (!IsMax) 3016 std::swap(CstLow, CstHigh); 3017 3018 const APInt &ValueLow = CstLow->getAPIntValue(); 3019 const APInt &ValueHigh = CstHigh->getAPIntValue(); 3020 if (ValueLow.sle(ValueHigh)) { 3021 unsigned LowSignBits = ValueLow.getNumSignBits(); 3022 unsigned HighSignBits = ValueHigh.getNumSignBits(); 3023 unsigned MinSignBits = std::min(LowSignBits, HighSignBits); 3024 if (ValueLow.isNegative() && ValueHigh.isNegative()) { 3025 Known.One.setHighBits(MinSignBits); 3026 break; 3027 } 3028 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) { 3029 Known.Zero.setHighBits(MinSignBits); 3030 break; 3031 } 3032 } 3033 } 3034 3035 // Fallback - just get the shared known bits of the operands. 3036 computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); 3037 if (Known.isUnknown()) break; // Early-out 3038 computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1); 3039 Known.Zero &= Known2.Zero; 3040 Known.One &= Known2.One; 3041 break; 3042 } 3043 case ISD::FrameIndex: 3044 case ISD::TargetFrameIndex: 3045 TLI->computeKnownBitsForFrameIndex(Op, Known, DemandedElts, *this, Depth); 3046 break; 3047 3048 default: 3049 if (Opcode < ISD::BUILTIN_OP_END) 3050 break; 3051 LLVM_FALLTHROUGH; 3052 case ISD::INTRINSIC_WO_CHAIN: 3053 case ISD::INTRINSIC_W_CHAIN: 3054 case ISD::INTRINSIC_VOID: 3055 // Allow the target to implement this method for its nodes. 3056 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth); 3057 break; 3058 } 3059 3060 assert(!Known.hasConflict() && "Bits known to be one AND zero?"); 3061 } 3062 3063 SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0, 3064 SDValue N1) const { 3065 // X + 0 never overflow 3066 if (isNullConstant(N1)) 3067 return OFK_Never; 3068 3069 KnownBits N1Known; 3070 computeKnownBits(N1, N1Known); 3071 if (N1Known.Zero.getBoolValue()) { 3072 KnownBits N0Known; 3073 computeKnownBits(N0, N0Known); 3074 3075 bool overflow; 3076 (void)(~N0Known.Zero).uadd_ov(~N1Known.Zero, overflow); 3077 if (!overflow) 3078 return OFK_Never; 3079 } 3080 3081 // mulhi + 1 never overflow 3082 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 && 3083 (~N1Known.Zero & 0x01) == ~N1Known.Zero) 3084 return OFK_Never; 3085 3086 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1) { 3087 KnownBits N0Known; 3088 computeKnownBits(N0, N0Known); 3089 3090 if ((~N0Known.Zero & 0x01) == ~N0Known.Zero) 3091 return OFK_Never; 3092 } 3093 3094 return OFK_Sometime; 3095 } 3096 3097 bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const { 3098 EVT OpVT = Val.getValueType(); 3099 unsigned BitWidth = OpVT.getScalarSizeInBits(); 3100 3101 // Is the constant a known power of 2? 3102 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val)) 3103 return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2(); 3104 3105 // A left-shift of a constant one will have exactly one bit set because 3106 // shifting the bit off the end is undefined. 3107 if (Val.getOpcode() == ISD::SHL) { 3108 auto *C = isConstOrConstSplat(Val.getOperand(0)); 3109 if (C && C->getAPIntValue() == 1) 3110 return true; 3111 } 3112 3113 // Similarly, a logical right-shift of a constant sign-bit will have exactly 3114 // one bit set. 3115 if (Val.getOpcode() == ISD::SRL) { 3116 auto *C = isConstOrConstSplat(Val.getOperand(0)); 3117 if (C && C->getAPIntValue().isSignMask()) 3118 return true; 3119 } 3120 3121 // Are all operands of a build vector constant powers of two? 3122 if (Val.getOpcode() == ISD::BUILD_VECTOR) 3123 if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) { 3124 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E)) 3125 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2(); 3126 return false; 3127 })) 3128 return true; 3129 3130 // More could be done here, though the above checks are enough 3131 // to handle some common cases. 3132 3133 // Fall back to computeKnownBits to catch other known cases. 3134 KnownBits Known; 3135 computeKnownBits(Val, Known); 3136 return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1); 3137 } 3138 3139 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const { 3140 EVT VT = Op.getValueType(); 3141 APInt DemandedElts = VT.isVector() 3142 ? APInt::getAllOnesValue(VT.getVectorNumElements()) 3143 : APInt(1, 1); 3144 return ComputeNumSignBits(Op, DemandedElts, Depth); 3145 } 3146 3147 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, 3148 unsigned Depth) const { 3149 EVT VT = Op.getValueType(); 3150 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!"); 3151 unsigned VTBits = VT.getScalarSizeInBits(); 3152 unsigned NumElts = DemandedElts.getBitWidth(); 3153 unsigned Tmp, Tmp2; 3154 unsigned FirstAnswer = 1; 3155 3156 if (auto *C = dyn_cast<ConstantSDNode>(Op)) { 3157 const APInt &Val = C->getAPIntValue(); 3158 return Val.getNumSignBits(); 3159 } 3160 3161 if (Depth == 6) 3162 return 1; // Limit search depth. 3163 3164 if (!DemandedElts) 3165 return 1; // No demanded elts, better to assume we don't know anything. 3166 3167 unsigned Opcode = Op.getOpcode(); 3168 switch (Opcode) { 3169 default: break; 3170 case ISD::AssertSext: 3171 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 3172 return VTBits-Tmp+1; 3173 case ISD::AssertZext: 3174 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 3175 return VTBits-Tmp; 3176 3177 case ISD::BUILD_VECTOR: 3178 Tmp = VTBits; 3179 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) { 3180 if (!DemandedElts[i]) 3181 continue; 3182 3183 SDValue SrcOp = Op.getOperand(i); 3184 Tmp2 = ComputeNumSignBits(Op.getOperand(i), Depth + 1); 3185 3186 // BUILD_VECTOR can implicitly truncate sources, we must handle this. 3187 if (SrcOp.getValueSizeInBits() != VTBits) { 3188 assert(SrcOp.getValueSizeInBits() > VTBits && 3189 "Expected BUILD_VECTOR implicit truncation"); 3190 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits; 3191 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1); 3192 } 3193 Tmp = std::min(Tmp, Tmp2); 3194 } 3195 return Tmp; 3196 3197 case ISD::VECTOR_SHUFFLE: { 3198 // Collect the minimum number of sign bits that are shared by every vector 3199 // element referenced by the shuffle. 3200 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0); 3201 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op); 3202 assert(NumElts == SVN->getMask().size() && "Unexpected vector size"); 3203 for (unsigned i = 0; i != NumElts; ++i) { 3204 int M = SVN->getMaskElt(i); 3205 if (!DemandedElts[i]) 3206 continue; 3207 // For UNDEF elements, we don't know anything about the common state of 3208 // the shuffle result. 3209 if (M < 0) 3210 return 1; 3211 if ((unsigned)M < NumElts) 3212 DemandedLHS.setBit((unsigned)M % NumElts); 3213 else 3214 DemandedRHS.setBit((unsigned)M % NumElts); 3215 } 3216 Tmp = std::numeric_limits<unsigned>::max(); 3217 if (!!DemandedLHS) 3218 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1); 3219 if (!!DemandedRHS) { 3220 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1); 3221 Tmp = std::min(Tmp, Tmp2); 3222 } 3223 // If we don't know anything, early out and try computeKnownBits fall-back. 3224 if (Tmp == 1) 3225 break; 3226 assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); 3227 return Tmp; 3228 } 3229 3230 case ISD::BITCAST: { 3231 SDValue N0 = Op.getOperand(0); 3232 EVT SrcVT = N0.getValueType(); 3233 unsigned SrcBits = SrcVT.getScalarSizeInBits(); 3234 3235 // Ignore bitcasts from unsupported types.. 3236 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint())) 3237 break; 3238 3239 // Fast handling of 'identity' bitcasts. 3240 if (VTBits == SrcBits) 3241 return ComputeNumSignBits(N0, DemandedElts, Depth + 1); 3242 3243 // Bitcast 'large element' scalar/vector to 'small element' vector. 3244 // TODO: Handle cases other than 'sign splat' when we have a use case. 3245 // Requires handling of DemandedElts and Endianness. 3246 if ((SrcBits % VTBits) == 0) { 3247 assert(Op.getValueType().isVector() && "Expected bitcast to vector"); 3248 Tmp = ComputeNumSignBits(N0, Depth + 1); 3249 if (Tmp == SrcBits) 3250 return VTBits; 3251 } 3252 break; 3253 } 3254 3255 case ISD::SIGN_EXTEND: 3256 Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits(); 3257 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp; 3258 case ISD::SIGN_EXTEND_INREG: 3259 // Max of the input and what this extends. 3260 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits(); 3261 Tmp = VTBits-Tmp+1; 3262 Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); 3263 return std::max(Tmp, Tmp2); 3264 case ISD::SIGN_EXTEND_VECTOR_INREG: { 3265 SDValue Src = Op.getOperand(0); 3266 EVT SrcVT = Src.getValueType(); 3267 APInt DemandedSrcElts = DemandedElts.zext(SrcVT.getVectorNumElements()); 3268 Tmp = VTBits - SrcVT.getScalarSizeInBits(); 3269 return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp; 3270 } 3271 3272 case ISD::SRA: 3273 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); 3274 // SRA X, C -> adds C sign bits. 3275 if (ConstantSDNode *C = 3276 isConstOrDemandedConstSplat(Op.getOperand(1), DemandedElts)) { 3277 APInt ShiftVal = C->getAPIntValue(); 3278 ShiftVal += Tmp; 3279 Tmp = ShiftVal.uge(VTBits) ? VTBits : ShiftVal.getZExtValue(); 3280 } 3281 return Tmp; 3282 case ISD::SHL: 3283 if (ConstantSDNode *C = 3284 isConstOrDemandedConstSplat(Op.getOperand(1), DemandedElts)) { 3285 // shl destroys sign bits. 3286 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); 3287 if (C->getAPIntValue().uge(VTBits) || // Bad shift. 3288 C->getAPIntValue().uge(Tmp)) break; // Shifted all sign bits out. 3289 return Tmp - C->getZExtValue(); 3290 } 3291 break; 3292 case ISD::AND: 3293 case ISD::OR: 3294 case ISD::XOR: // NOT is handled here. 3295 // Logical binary ops preserve the number of sign bits at the worst. 3296 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); 3297 if (Tmp != 1) { 3298 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1); 3299 FirstAnswer = std::min(Tmp, Tmp2); 3300 // We computed what we know about the sign bits as our first 3301 // answer. Now proceed to the generic code that uses 3302 // computeKnownBits, and pick whichever answer is better. 3303 } 3304 break; 3305 3306 case ISD::SELECT: 3307 case ISD::VSELECT: 3308 Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1); 3309 if (Tmp == 1) return 1; // Early out. 3310 Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1); 3311 return std::min(Tmp, Tmp2); 3312 case ISD::SELECT_CC: 3313 Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1); 3314 if (Tmp == 1) return 1; // Early out. 3315 Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1); 3316 return std::min(Tmp, Tmp2); 3317 3318 case ISD::SMIN: 3319 case ISD::SMAX: { 3320 // If we have a clamp pattern, we know that the number of sign bits will be 3321 // the minimum of the clamp min/max range. 3322 bool IsMax = (Opcode == ISD::SMAX); 3323 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr; 3324 if ((CstLow = isConstOrDemandedConstSplat(Op.getOperand(1), DemandedElts))) 3325 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) 3326 CstHigh = isConstOrDemandedConstSplat(Op.getOperand(0).getOperand(1), 3327 DemandedElts); 3328 if (CstLow && CstHigh) { 3329 if (!IsMax) 3330 std::swap(CstLow, CstHigh); 3331 if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) { 3332 Tmp = CstLow->getAPIntValue().getNumSignBits(); 3333 Tmp2 = CstHigh->getAPIntValue().getNumSignBits(); 3334 return std::min(Tmp, Tmp2); 3335 } 3336 } 3337 3338 // Fallback - just get the minimum number of sign bits of the operands. 3339 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1); 3340 if (Tmp == 1) 3341 return 1; // Early out. 3342 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1); 3343 return std::min(Tmp, Tmp2); 3344 } 3345 case ISD::UMIN: 3346 case ISD::UMAX: 3347 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1); 3348 if (Tmp == 1) 3349 return 1; // Early out. 3350 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1); 3351 return std::min(Tmp, Tmp2); 3352 case ISD::SADDO: 3353 case ISD::UADDO: 3354 case ISD::SSUBO: 3355 case ISD::USUBO: 3356 case ISD::SMULO: 3357 case ISD::UMULO: 3358 if (Op.getResNo() != 1) 3359 break; 3360 // The boolean result conforms to getBooleanContents. Fall through. 3361 // If setcc returns 0/-1, all bits are sign bits. 3362 // We know that we have an integer-based boolean since these operations 3363 // are only available for integer. 3364 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) == 3365 TargetLowering::ZeroOrNegativeOneBooleanContent) 3366 return VTBits; 3367 break; 3368 case ISD::SETCC: 3369 // If setcc returns 0/-1, all bits are sign bits. 3370 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == 3371 TargetLowering::ZeroOrNegativeOneBooleanContent) 3372 return VTBits; 3373 break; 3374 case ISD::ROTL: 3375 case ISD::ROTR: 3376 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 3377 unsigned RotAmt = C->getAPIntValue().urem(VTBits); 3378 3379 // Handle rotate right by N like a rotate left by 32-N. 3380 if (Opcode == ISD::ROTR) 3381 RotAmt = (VTBits - RotAmt) % VTBits; 3382 3383 // If we aren't rotating out all of the known-in sign bits, return the 3384 // number that are left. This handles rotl(sext(x), 1) for example. 3385 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 3386 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt); 3387 } 3388 break; 3389 case ISD::ADD: 3390 case ISD::ADDC: 3391 // Add can have at most one carry bit. Thus we know that the output 3392 // is, at worst, one more bit than the inputs. 3393 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 3394 if (Tmp == 1) return 1; // Early out. 3395 3396 // Special case decrementing a value (ADD X, -1): 3397 if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 3398 if (CRHS->isAllOnesValue()) { 3399 KnownBits Known; 3400 computeKnownBits(Op.getOperand(0), Known, Depth+1); 3401 3402 // If the input is known to be 0 or 1, the output is 0/-1, which is all 3403 // sign bits set. 3404 if ((Known.Zero | 1).isAllOnesValue()) 3405 return VTBits; 3406 3407 // If we are subtracting one from a positive number, there is no carry 3408 // out of the result. 3409 if (Known.isNonNegative()) 3410 return Tmp; 3411 } 3412 3413 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 3414 if (Tmp2 == 1) return 1; 3415 return std::min(Tmp, Tmp2)-1; 3416 3417 case ISD::SUB: 3418 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 3419 if (Tmp2 == 1) return 1; 3420 3421 // Handle NEG. 3422 if (ConstantSDNode *CLHS = isConstOrConstSplat(Op.getOperand(0))) 3423 if (CLHS->isNullValue()) { 3424 KnownBits Known; 3425 computeKnownBits(Op.getOperand(1), Known, Depth+1); 3426 // If the input is known to be 0 or 1, the output is 0/-1, which is all 3427 // sign bits set. 3428 if ((Known.Zero | 1).isAllOnesValue()) 3429 return VTBits; 3430 3431 // If the input is known to be positive (the sign bit is known clear), 3432 // the output of the NEG has the same number of sign bits as the input. 3433 if (Known.isNonNegative()) 3434 return Tmp2; 3435 3436 // Otherwise, we treat this like a SUB. 3437 } 3438 3439 // Sub can have at most one carry bit. Thus we know that the output 3440 // is, at worst, one more bit than the inputs. 3441 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 3442 if (Tmp == 1) return 1; // Early out. 3443 return std::min(Tmp, Tmp2)-1; 3444 case ISD::TRUNCATE: { 3445 // Check if the sign bits of source go down as far as the truncated value. 3446 unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits(); 3447 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1); 3448 if (NumSrcSignBits > (NumSrcBits - VTBits)) 3449 return NumSrcSignBits - (NumSrcBits - VTBits); 3450 break; 3451 } 3452 case ISD::EXTRACT_ELEMENT: { 3453 const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1); 3454 const int BitWidth = Op.getValueSizeInBits(); 3455 const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth; 3456 3457 // Get reverse index (starting from 1), Op1 value indexes elements from 3458 // little end. Sign starts at big end. 3459 const int rIndex = Items - 1 - Op.getConstantOperandVal(1); 3460 3461 // If the sign portion ends in our element the subtraction gives correct 3462 // result. Otherwise it gives either negative or > bitwidth result 3463 return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0); 3464 } 3465 case ISD::INSERT_VECTOR_ELT: { 3466 SDValue InVec = Op.getOperand(0); 3467 SDValue InVal = Op.getOperand(1); 3468 SDValue EltNo = Op.getOperand(2); 3469 unsigned NumElts = InVec.getValueType().getVectorNumElements(); 3470 3471 ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo); 3472 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) { 3473 // If we know the element index, split the demand between the 3474 // source vector and the inserted element. 3475 unsigned EltIdx = CEltNo->getZExtValue(); 3476 3477 // If we demand the inserted element then get its sign bits. 3478 Tmp = std::numeric_limits<unsigned>::max(); 3479 if (DemandedElts[EltIdx]) { 3480 // TODO - handle implicit truncation of inserted elements. 3481 if (InVal.getScalarValueSizeInBits() != VTBits) 3482 break; 3483 Tmp = ComputeNumSignBits(InVal, Depth + 1); 3484 } 3485 3486 // If we demand the source vector then get its sign bits, and determine 3487 // the minimum. 3488 APInt VectorElts = DemandedElts; 3489 VectorElts.clearBit(EltIdx); 3490 if (!!VectorElts) { 3491 Tmp2 = ComputeNumSignBits(InVec, VectorElts, Depth + 1); 3492 Tmp = std::min(Tmp, Tmp2); 3493 } 3494 } else { 3495 // Unknown element index, so ignore DemandedElts and demand them all. 3496 Tmp = ComputeNumSignBits(InVec, Depth + 1); 3497 Tmp2 = ComputeNumSignBits(InVal, Depth + 1); 3498 Tmp = std::min(Tmp, Tmp2); 3499 } 3500 assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); 3501 return Tmp; 3502 } 3503 case ISD::EXTRACT_VECTOR_ELT: { 3504 SDValue InVec = Op.getOperand(0); 3505 SDValue EltNo = Op.getOperand(1); 3506 EVT VecVT = InVec.getValueType(); 3507 const unsigned BitWidth = Op.getValueSizeInBits(); 3508 const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits(); 3509 const unsigned NumSrcElts = VecVT.getVectorNumElements(); 3510 3511 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know 3512 // anything about sign bits. But if the sizes match we can derive knowledge 3513 // about sign bits from the vector operand. 3514 if (BitWidth != EltBitWidth) 3515 break; 3516 3517 // If we know the element index, just demand that vector element, else for 3518 // an unknown element index, ignore DemandedElts and demand them all. 3519 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts); 3520 ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo); 3521 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)) 3522 DemandedSrcElts = 3523 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue()); 3524 3525 return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1); 3526 } 3527 case ISD::EXTRACT_SUBVECTOR: { 3528 // If we know the element index, just demand that subvector elements, 3529 // otherwise demand them all. 3530 SDValue Src = Op.getOperand(0); 3531 ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 3532 unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); 3533 if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) { 3534 // Offset the demanded elts by the subvector index. 3535 uint64_t Idx = SubIdx->getZExtValue(); 3536 APInt DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx); 3537 return ComputeNumSignBits(Src, DemandedSrc, Depth + 1); 3538 } 3539 return ComputeNumSignBits(Src, Depth + 1); 3540 } 3541 case ISD::CONCAT_VECTORS: 3542 // Determine the minimum number of sign bits across all demanded 3543 // elts of the input vectors. Early out if the result is already 1. 3544 Tmp = std::numeric_limits<unsigned>::max(); 3545 EVT SubVectorVT = Op.getOperand(0).getValueType(); 3546 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements(); 3547 unsigned NumSubVectors = Op.getNumOperands(); 3548 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) { 3549 APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts); 3550 DemandedSub = DemandedSub.trunc(NumSubVectorElts); 3551 if (!DemandedSub) 3552 continue; 3553 Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1); 3554 Tmp = std::min(Tmp, Tmp2); 3555 } 3556 assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); 3557 return Tmp; 3558 } 3559 3560 // If we are looking at the loaded value of the SDNode. 3561 if (Op.getResNo() == 0) { 3562 // Handle LOADX separately here. EXTLOAD case will fallthrough. 3563 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) { 3564 unsigned ExtType = LD->getExtensionType(); 3565 switch (ExtType) { 3566 default: break; 3567 case ISD::SEXTLOAD: // '17' bits known 3568 Tmp = LD->getMemoryVT().getScalarSizeInBits(); 3569 return VTBits-Tmp+1; 3570 case ISD::ZEXTLOAD: // '16' bits known 3571 Tmp = LD->getMemoryVT().getScalarSizeInBits(); 3572 return VTBits-Tmp; 3573 } 3574 } 3575 } 3576 3577 // Allow the target to implement this method for its nodes. 3578 if (Opcode >= ISD::BUILTIN_OP_END || 3579 Opcode == ISD::INTRINSIC_WO_CHAIN || 3580 Opcode == ISD::INTRINSIC_W_CHAIN || 3581 Opcode == ISD::INTRINSIC_VOID) { 3582 unsigned NumBits = 3583 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth); 3584 if (NumBits > 1) 3585 FirstAnswer = std::max(FirstAnswer, NumBits); 3586 } 3587 3588 // Finally, if we can prove that the top bits of the result are 0's or 1's, 3589 // use this information. 3590 KnownBits Known; 3591 computeKnownBits(Op, Known, DemandedElts, Depth); 3592 3593 APInt Mask; 3594 if (Known.isNonNegative()) { // sign bit is 0 3595 Mask = Known.Zero; 3596 } else if (Known.isNegative()) { // sign bit is 1; 3597 Mask = Known.One; 3598 } else { 3599 // Nothing known. 3600 return FirstAnswer; 3601 } 3602 3603 // Okay, we know that the sign bit in Mask is set. Use CLZ to determine 3604 // the number of identical bits in the top of the input value. 3605 Mask = ~Mask; 3606 Mask <<= Mask.getBitWidth()-VTBits; 3607 // Return # leading zeros. We use 'min' here in case Val was zero before 3608 // shifting. We don't want to return '64' as for an i32 "0". 3609 return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros())); 3610 } 3611 3612 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { 3613 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || 3614 !isa<ConstantSDNode>(Op.getOperand(1))) 3615 return false; 3616 3617 if (Op.getOpcode() == ISD::OR && 3618 !MaskedValueIsZero(Op.getOperand(0), 3619 cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue())) 3620 return false; 3621 3622 return true; 3623 } 3624 3625 bool SelectionDAG::isKnownNeverNaN(SDValue Op) const { 3626 // If we're told that NaNs won't happen, assume they won't. 3627 if (getTarget().Options.NoNaNsFPMath) 3628 return true; 3629 3630 if (Op->getFlags().hasNoNaNs()) 3631 return true; 3632 3633 // If the value is a constant, we can obviously see if it is a NaN or not. 3634 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 3635 return !C->getValueAPF().isNaN(); 3636 3637 // TODO: Recognize more cases here. 3638 3639 return false; 3640 } 3641 3642 bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const { 3643 assert(Op.getValueType().isFloatingPoint() && 3644 "Floating point type expected"); 3645 3646 // If the value is a constant, we can obviously see if it is a zero or not. 3647 // TODO: Add BuildVector support. 3648 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 3649 return !C->isZero(); 3650 return false; 3651 } 3652 3653 bool SelectionDAG::isKnownNeverZero(SDValue Op) const { 3654 assert(!Op.getValueType().isFloatingPoint() && 3655 "Floating point types unsupported - use isKnownNeverZeroFloat"); 3656 3657 // If the value is a constant, we can obviously see if it is a zero or not. 3658 if (ISD::matchUnaryPredicate( 3659 Op, [](ConstantSDNode *C) { return !C->isNullValue(); })) 3660 return true; 3661 3662 // TODO: Recognize more cases here. 3663 switch (Op.getOpcode()) { 3664 default: break; 3665 case ISD::OR: 3666 if (isKnownNeverZero(Op.getOperand(1)) || 3667 isKnownNeverZero(Op.getOperand(0))) 3668 return true; 3669 break; 3670 } 3671 3672 return false; 3673 } 3674 3675 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { 3676 // Check the obvious case. 3677 if (A == B) return true; 3678 3679 // For for negative and positive zero. 3680 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A)) 3681 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B)) 3682 if (CA->isZero() && CB->isZero()) return true; 3683 3684 // Otherwise they may not be equal. 3685 return false; 3686 } 3687 3688 // FIXME: unify with llvm::haveNoCommonBitsSet. 3689 // FIXME: could also handle masked merge pattern (X & ~M) op (Y & M) 3690 bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const { 3691 assert(A.getValueType() == B.getValueType() && 3692 "Values must have the same type"); 3693 KnownBits AKnown, BKnown; 3694 computeKnownBits(A, AKnown); 3695 computeKnownBits(B, BKnown); 3696 return (AKnown.Zero | BKnown.Zero).isAllOnesValue(); 3697 } 3698 3699 static SDValue FoldCONCAT_VECTORS(const SDLoc &DL, EVT VT, 3700 ArrayRef<SDValue> Ops, 3701 SelectionDAG &DAG) { 3702 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!"); 3703 assert(llvm::all_of(Ops, 3704 [Ops](SDValue Op) { 3705 return Ops[0].getValueType() == Op.getValueType(); 3706 }) && 3707 "Concatenation of vectors with inconsistent value types!"); 3708 assert((Ops.size() * Ops[0].getValueType().getVectorNumElements()) == 3709 VT.getVectorNumElements() && 3710 "Incorrect element count in vector concatenation!"); 3711 3712 if (Ops.size() == 1) 3713 return Ops[0]; 3714 3715 // Concat of UNDEFs is UNDEF. 3716 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); })) 3717 return DAG.getUNDEF(VT); 3718 3719 // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be 3720 // simplified to one big BUILD_VECTOR. 3721 // FIXME: Add support for SCALAR_TO_VECTOR as well. 3722 EVT SVT = VT.getScalarType(); 3723 SmallVector<SDValue, 16> Elts; 3724 for (SDValue Op : Ops) { 3725 EVT OpVT = Op.getValueType(); 3726 if (Op.isUndef()) 3727 Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT)); 3728 else if (Op.getOpcode() == ISD::BUILD_VECTOR) 3729 Elts.append(Op->op_begin(), Op->op_end()); 3730 else 3731 return SDValue(); 3732 } 3733 3734 // BUILD_VECTOR requires all inputs to be of the same type, find the 3735 // maximum type and extend them all. 3736 for (SDValue Op : Elts) 3737 SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT); 3738 3739 if (SVT.bitsGT(VT.getScalarType())) 3740 for (SDValue &Op : Elts) 3741 Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT) 3742 ? DAG.getZExtOrTrunc(Op, DL, SVT) 3743 : DAG.getSExtOrTrunc(Op, DL, SVT); 3744 3745 SDValue V = DAG.getBuildVector(VT, DL, Elts); 3746 NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG); 3747 return V; 3748 } 3749 3750 /// Gets or creates the specified node. 3751 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) { 3752 FoldingSetNodeID ID; 3753 AddNodeIDNode(ID, Opcode, getVTList(VT), None); 3754 void *IP = nullptr; 3755 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) 3756 return SDValue(E, 0); 3757 3758 auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), 3759 getVTList(VT)); 3760 CSEMap.InsertNode(N, IP); 3761 3762 InsertNode(N); 3763 SDValue V = SDValue(N, 0); 3764 NewSDValueDbgMsg(V, "Creating new node: ", this); 3765 return V; 3766 } 3767 3768 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 3769 SDValue Operand, const SDNodeFlags Flags) { 3770 // Constant fold unary operations with an integer constant operand. Even 3771 // opaque constant will be folded, because the folding of unary operations 3772 // doesn't create new constants with different values. Nevertheless, the 3773 // opaque flag is preserved during folding to prevent future folding with 3774 // other constants. 3775 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) { 3776 const APInt &Val = C->getAPIntValue(); 3777 switch (Opcode) { 3778 default: break; 3779 case ISD::SIGN_EXTEND: 3780 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT, 3781 C->isTargetOpcode(), C->isOpaque()); 3782 case ISD::ANY_EXTEND: 3783 case ISD::ZERO_EXTEND: 3784 case ISD::TRUNCATE: 3785 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT, 3786 C->isTargetOpcode(), C->isOpaque()); 3787 case ISD::UINT_TO_FP: 3788 case ISD::SINT_TO_FP: { 3789 APFloat apf(EVTToAPFloatSemantics(VT), 3790 APInt::getNullValue(VT.getSizeInBits())); 3791 (void)apf.convertFromAPInt(Val, 3792 Opcode==ISD::SINT_TO_FP, 3793 APFloat::rmNearestTiesToEven); 3794 return getConstantFP(apf, DL, VT); 3795 } 3796 case ISD::BITCAST: 3797 if (VT == MVT::f16 && C->getValueType(0) == MVT::i16) 3798 return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT); 3799 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) 3800 return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT); 3801 if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) 3802 return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT); 3803 if (VT == MVT::f128 && C->getValueType(0) == MVT::i128) 3804 return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT); 3805 break; 3806 case ISD::ABS: 3807 return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(), 3808 C->isOpaque()); 3809 case ISD::BITREVERSE: 3810 return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(), 3811 C->isOpaque()); 3812 case ISD::BSWAP: 3813 return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(), 3814 C->isOpaque()); 3815 case ISD::CTPOP: 3816 return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(), 3817 C->isOpaque()); 3818 case ISD::CTLZ: 3819 case ISD::CTLZ_ZERO_UNDEF: 3820 return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(), 3821 C->isOpaque()); 3822 case ISD::CTTZ: 3823 case ISD::CTTZ_ZERO_UNDEF: 3824 return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(), 3825 C->isOpaque()); 3826 case ISD::FP16_TO_FP: { 3827 bool Ignored; 3828 APFloat FPV(APFloat::IEEEhalf(), 3829 (Val.getBitWidth() == 16) ? Val : Val.trunc(16)); 3830 3831 // This can return overflow, underflow, or inexact; we don't care. 3832 // FIXME need to be more flexible about rounding mode. 3833 (void)FPV.convert(EVTToAPFloatSemantics(VT), 3834 APFloat::rmNearestTiesToEven, &Ignored); 3835 return getConstantFP(FPV, DL, VT); 3836 } 3837 } 3838 } 3839 3840 // Constant fold unary operations with a floating point constant operand. 3841 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) { 3842 APFloat V = C->getValueAPF(); // make copy 3843 switch (Opcode) { 3844 case ISD::FNEG: 3845 V.changeSign(); 3846 return getConstantFP(V, DL, VT); 3847 case ISD::FABS: 3848 V.clearSign(); 3849 return getConstantFP(V, DL, VT); 3850 case ISD::FCEIL: { 3851 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive); 3852 if (fs == APFloat::opOK || fs == APFloat::opInexact) 3853 return getConstantFP(V, DL, VT); 3854 break; 3855 } 3856 case ISD::FTRUNC: { 3857 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero); 3858 if (fs == APFloat::opOK || fs == APFloat::opInexact) 3859 return getConstantFP(V, DL, VT); 3860 break; 3861 } 3862 case ISD::FFLOOR: { 3863 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative); 3864 if (fs == APFloat::opOK || fs == APFloat::opInexact) 3865 return getConstantFP(V, DL, VT); 3866 break; 3867 } 3868 case ISD::FP_EXTEND: { 3869 bool ignored; 3870 // This can return overflow, underflow, or inexact; we don't care. 3871 // FIXME need to be more flexible about rounding mode. 3872 (void)V.convert(EVTToAPFloatSemantics(VT), 3873 APFloat::rmNearestTiesToEven, &ignored); 3874 return getConstantFP(V, DL, VT); 3875 } 3876 case ISD::FP_TO_SINT: 3877 case ISD::FP_TO_UINT: { 3878 bool ignored; 3879 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT); 3880 // FIXME need to be more flexible about rounding mode. 3881 APFloat::opStatus s = 3882 V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored); 3883 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual 3884 break; 3885 return getConstant(IntVal, DL, VT); 3886 } 3887 case ISD::BITCAST: 3888 if (VT == MVT::i16 && C->getValueType(0) == MVT::f16) 3889 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT); 3890 else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) 3891 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT); 3892 else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) 3893 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT); 3894 break; 3895 case ISD::FP_TO_FP16: { 3896 bool Ignored; 3897 // This can return overflow, underflow, or inexact; we don't care. 3898 // FIXME need to be more flexible about rounding mode. 3899 (void)V.convert(APFloat::IEEEhalf(), 3900 APFloat::rmNearestTiesToEven, &Ignored); 3901 return getConstant(V.bitcastToAPInt(), DL, VT); 3902 } 3903 } 3904 } 3905 3906 // Constant fold unary operations with a vector integer or float operand. 3907 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand)) { 3908 if (BV->isConstant()) { 3909 switch (Opcode) { 3910 default: 3911 // FIXME: Entirely reasonable to perform folding of other unary 3912 // operations here as the need arises. 3913 break; 3914 case ISD::FNEG: 3915 case ISD::FABS: 3916 case ISD::FCEIL: 3917 case ISD::FTRUNC: 3918 case ISD::FFLOOR: 3919 case ISD::FP_EXTEND: 3920 case ISD::FP_TO_SINT: 3921 case ISD::FP_TO_UINT: 3922 case ISD::TRUNCATE: 3923 case ISD::ANY_EXTEND: 3924 case ISD::ZERO_EXTEND: 3925 case ISD::SIGN_EXTEND: 3926 case ISD::UINT_TO_FP: 3927 case ISD::SINT_TO_FP: 3928 case ISD::ABS: 3929 case ISD::BITREVERSE: 3930 case ISD::BSWAP: 3931 case ISD::CTLZ: 3932 case ISD::CTLZ_ZERO_UNDEF: 3933 case ISD::CTTZ: 3934 case ISD::CTTZ_ZERO_UNDEF: 3935 case ISD::CTPOP: { 3936 SDValue Ops = { Operand }; 3937 if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) 3938 return Fold; 3939 } 3940 } 3941 } 3942 } 3943 3944 unsigned OpOpcode = Operand.getNode()->getOpcode(); 3945 switch (Opcode) { 3946 case ISD::TokenFactor: 3947 case ISD::MERGE_VALUES: 3948 case ISD::CONCAT_VECTORS: 3949 return Operand; // Factor, merge or concat of one node? No need. 3950 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); 3951 case ISD::FP_EXTEND: 3952 assert(VT.isFloatingPoint() && 3953 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); 3954 if (Operand.getValueType() == VT) return Operand; // noop conversion. 3955 assert((!VT.isVector() || 3956 VT.getVectorNumElements() == 3957 Operand.getValueType().getVectorNumElements()) && 3958 "Vector element count mismatch!"); 3959 assert(Operand.getValueType().bitsLT(VT) && 3960 "Invalid fpext node, dst < src!"); 3961 if (Operand.isUndef()) 3962 return getUNDEF(VT); 3963 break; 3964 case ISD::SIGN_EXTEND: 3965 assert(VT.isInteger() && Operand.getValueType().isInteger() && 3966 "Invalid SIGN_EXTEND!"); 3967 if (Operand.getValueType() == VT) return Operand; // noop extension 3968 assert((!VT.isVector() || 3969 VT.getVectorNumElements() == 3970 Operand.getValueType().getVectorNumElements()) && 3971 "Vector element count mismatch!"); 3972 assert(Operand.getValueType().bitsLT(VT) && 3973 "Invalid sext node, dst < src!"); 3974 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) 3975 return getNode(OpOpcode, DL, VT, Operand.getOperand(0)); 3976 else if (OpOpcode == ISD::UNDEF) 3977 // sext(undef) = 0, because the top bits will all be the same. 3978 return getConstant(0, DL, VT); 3979 break; 3980 case ISD::ZERO_EXTEND: 3981 assert(VT.isInteger() && Operand.getValueType().isInteger() && 3982 "Invalid ZERO_EXTEND!"); 3983 if (Operand.getValueType() == VT) return Operand; // noop extension 3984 assert((!VT.isVector() || 3985 VT.getVectorNumElements() == 3986 Operand.getValueType().getVectorNumElements()) && 3987 "Vector element count mismatch!"); 3988 assert(Operand.getValueType().bitsLT(VT) && 3989 "Invalid zext node, dst < src!"); 3990 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) 3991 return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0)); 3992 else if (OpOpcode == ISD::UNDEF) 3993 // zext(undef) = 0, because the top bits will be zero. 3994 return getConstant(0, DL, VT); 3995 break; 3996 case ISD::ANY_EXTEND: 3997 assert(VT.isInteger() && Operand.getValueType().isInteger() && 3998 "Invalid ANY_EXTEND!"); 3999 if (Operand.getValueType() == VT) return Operand; // noop extension 4000 assert((!VT.isVector() || 4001 VT.getVectorNumElements() == 4002 Operand.getValueType().getVectorNumElements()) && 4003 "Vector element count mismatch!"); 4004 assert(Operand.getValueType().bitsLT(VT) && 4005 "Invalid anyext node, dst < src!"); 4006 4007 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 4008 OpOpcode == ISD::ANY_EXTEND) 4009 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) 4010 return getNode(OpOpcode, DL, VT, Operand.getOperand(0)); 4011 else if (OpOpcode == ISD::UNDEF) 4012 return getUNDEF(VT); 4013 4014 // (ext (trunc x)) -> x 4015 if (OpOpcode == ISD::TRUNCATE) { 4016 SDValue OpOp = Operand.getOperand(0); 4017 if (OpOp.getValueType() == VT) { 4018 transferDbgValues(Operand, OpOp); 4019 return OpOp; 4020 } 4021 } 4022 break; 4023 case ISD::TRUNCATE: 4024 assert(VT.isInteger() && Operand.getValueType().isInteger() && 4025 "Invalid TRUNCATE!"); 4026 if (Operand.getValueType() == VT) return Operand; // noop truncate 4027 assert((!VT.isVector() || 4028 VT.getVectorNumElements() == 4029 Operand.getValueType().getVectorNumElements()) && 4030 "Vector element count mismatch!"); 4031 assert(Operand.getValueType().bitsGT(VT) && 4032 "Invalid truncate node, src < dst!"); 4033 if (OpOpcode == ISD::TRUNCATE) 4034 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0)); 4035 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 4036 OpOpcode == ISD::ANY_EXTEND) { 4037 // If the source is smaller than the dest, we still need an extend. 4038 if (Operand.getOperand(0).getValueType().getScalarType() 4039 .bitsLT(VT.getScalarType())) 4040 return getNode(OpOpcode, DL, VT, Operand.getOperand(0)); 4041 if (Operand.getOperand(0).getValueType().bitsGT(VT)) 4042 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0)); 4043 return Operand.getOperand(0); 4044 } 4045 if (OpOpcode == ISD::UNDEF) 4046 return getUNDEF(VT); 4047 break; 4048 case ISD::ABS: 4049 assert(VT.isInteger() && VT == Operand.getValueType() && 4050 "Invalid ABS!"); 4051 if (OpOpcode == ISD::UNDEF) 4052 return getUNDEF(VT); 4053 break; 4054 case ISD::BSWAP: 4055 assert(VT.isInteger() && VT == Operand.getValueType() && 4056 "Invalid BSWAP!"); 4057 assert((VT.getScalarSizeInBits() % 16 == 0) && 4058 "BSWAP types must be a multiple of 16 bits!"); 4059 if (OpOpcode == ISD::UNDEF) 4060 return getUNDEF(VT); 4061 break; 4062 case ISD::BITREVERSE: 4063 assert(VT.isInteger() && VT == Operand.getValueType() && 4064 "Invalid BITREVERSE!"); 4065 if (OpOpcode == ISD::UNDEF) 4066 return getUNDEF(VT); 4067 break; 4068 case ISD::BITCAST: 4069 // Basic sanity checking. 4070 assert(VT.getSizeInBits() == Operand.getValueSizeInBits() && 4071 "Cannot BITCAST between types of different sizes!"); 4072 if (VT == Operand.getValueType()) return Operand; // noop conversion. 4073 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) 4074 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0)); 4075 if (OpOpcode == ISD::UNDEF) 4076 return getUNDEF(VT); 4077 break; 4078 case ISD::SCALAR_TO_VECTOR: 4079 assert(VT.isVector() && !Operand.getValueType().isVector() && 4080 (VT.getVectorElementType() == Operand.getValueType() || 4081 (VT.getVectorElementType().isInteger() && 4082 Operand.getValueType().isInteger() && 4083 VT.getVectorElementType().bitsLE(Operand.getValueType()))) && 4084 "Illegal SCALAR_TO_VECTOR node!"); 4085 if (OpOpcode == ISD::UNDEF) 4086 return getUNDEF(VT); 4087 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. 4088 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && 4089 isa<ConstantSDNode>(Operand.getOperand(1)) && 4090 Operand.getConstantOperandVal(1) == 0 && 4091 Operand.getOperand(0).getValueType() == VT) 4092 return Operand.getOperand(0); 4093 break; 4094 case ISD::FNEG: 4095 // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 4096 if ((getTarget().Options.UnsafeFPMath || Flags.hasNoSignedZeros()) && 4097 OpOpcode == ISD::FSUB) 4098 return getNode(ISD::FSUB, DL, VT, Operand.getOperand(1), 4099 Operand.getOperand(0), Flags); 4100 if (OpOpcode == ISD::FNEG) // --X -> X 4101 return Operand.getOperand(0); 4102 break; 4103 case ISD::FABS: 4104 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) 4105 return getNode(ISD::FABS, DL, VT, Operand.getOperand(0)); 4106 break; 4107 } 4108 4109 SDNode *N; 4110 SDVTList VTs = getVTList(VT); 4111 SDValue Ops[] = {Operand}; 4112 if (VT != MVT::Glue) { // Don't CSE flag producing nodes 4113 FoldingSetNodeID ID; 4114 AddNodeIDNode(ID, Opcode, VTs, Ops); 4115 void *IP = nullptr; 4116 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { 4117 E->intersectFlagsWith(Flags); 4118 return SDValue(E, 0); 4119 } 4120 4121 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 4122 N->setFlags(Flags); 4123 createOperands(N, Ops); 4124 CSEMap.InsertNode(N, IP); 4125 } else { 4126 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 4127 createOperands(N, Ops); 4128 } 4129 4130 InsertNode(N); 4131 SDValue V = SDValue(N, 0); 4132 NewSDValueDbgMsg(V, "Creating new node: ", this); 4133 return V; 4134 } 4135 4136 static std::pair<APInt, bool> FoldValue(unsigned Opcode, const APInt &C1, 4137 const APInt &C2) { 4138 switch (Opcode) { 4139 case ISD::ADD: return std::make_pair(C1 + C2, true); 4140 case ISD::SUB: return std::make_pair(C1 - C2, true); 4141 case ISD::MUL: return std::make_pair(C1 * C2, true); 4142 case ISD::AND: return std::make_pair(C1 & C2, true); 4143 case ISD::OR: return std::make_pair(C1 | C2, true); 4144 case ISD::XOR: return std::make_pair(C1 ^ C2, true); 4145 case ISD::SHL: return std::make_pair(C1 << C2, true); 4146 case ISD::SRL: return std::make_pair(C1.lshr(C2), true); 4147 case ISD::SRA: return std::make_pair(C1.ashr(C2), true); 4148 case ISD::ROTL: return std::make_pair(C1.rotl(C2), true); 4149 case ISD::ROTR: return std::make_pair(C1.rotr(C2), true); 4150 case ISD::SMIN: return std::make_pair(C1.sle(C2) ? C1 : C2, true); 4151 case ISD::SMAX: return std::make_pair(C1.sge(C2) ? C1 : C2, true); 4152 case ISD::UMIN: return std::make_pair(C1.ule(C2) ? C1 : C2, true); 4153 case ISD::UMAX: return std::make_pair(C1.uge(C2) ? C1 : C2, true); 4154 case ISD::UDIV: 4155 if (!C2.getBoolValue()) 4156 break; 4157 return std::make_pair(C1.udiv(C2), true); 4158 case ISD::UREM: 4159 if (!C2.getBoolValue()) 4160 break; 4161 return std::make_pair(C1.urem(C2), true); 4162 case ISD::SDIV: 4163 if (!C2.getBoolValue()) 4164 break; 4165 return std::make_pair(C1.sdiv(C2), true); 4166 case ISD::SREM: 4167 if (!C2.getBoolValue()) 4168 break; 4169 return std::make_pair(C1.srem(C2), true); 4170 } 4171 return std::make_pair(APInt(1, 0), false); 4172 } 4173 4174 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, 4175 EVT VT, const ConstantSDNode *Cst1, 4176 const ConstantSDNode *Cst2) { 4177 if (Cst1->isOpaque() || Cst2->isOpaque()) 4178 return SDValue(); 4179 4180 std::pair<APInt, bool> Folded = FoldValue(Opcode, Cst1->getAPIntValue(), 4181 Cst2->getAPIntValue()); 4182 if (!Folded.second) 4183 return SDValue(); 4184 return getConstant(Folded.first, DL, VT); 4185 } 4186 4187 SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT, 4188 const GlobalAddressSDNode *GA, 4189 const SDNode *N2) { 4190 if (GA->getOpcode() != ISD::GlobalAddress) 4191 return SDValue(); 4192 if (!TLI->isOffsetFoldingLegal(GA)) 4193 return SDValue(); 4194 const ConstantSDNode *Cst2 = dyn_cast<ConstantSDNode>(N2); 4195 if (!Cst2) 4196 return SDValue(); 4197 int64_t Offset = Cst2->getSExtValue(); 4198 switch (Opcode) { 4199 case ISD::ADD: break; 4200 case ISD::SUB: Offset = -uint64_t(Offset); break; 4201 default: return SDValue(); 4202 } 4203 return getGlobalAddress(GA->getGlobal(), SDLoc(Cst2), VT, 4204 GA->getOffset() + uint64_t(Offset)); 4205 } 4206 4207 bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) { 4208 switch (Opcode) { 4209 case ISD::SDIV: 4210 case ISD::UDIV: 4211 case ISD::SREM: 4212 case ISD::UREM: { 4213 // If a divisor is zero/undef or any element of a divisor vector is 4214 // zero/undef, the whole op is undef. 4215 assert(Ops.size() == 2 && "Div/rem should have 2 operands"); 4216 SDValue Divisor = Ops[1]; 4217 if (Divisor.isUndef() || isNullConstant(Divisor)) 4218 return true; 4219 4220 return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) && 4221 llvm::any_of(Divisor->op_values(), 4222 [](SDValue V) { return V.isUndef() || 4223 isNullConstant(V); }); 4224 // TODO: Handle signed overflow. 4225 } 4226 // TODO: Handle oversized shifts. 4227 default: 4228 return false; 4229 } 4230 } 4231 4232 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, 4233 EVT VT, SDNode *Cst1, 4234 SDNode *Cst2) { 4235 // If the opcode is a target-specific ISD node, there's nothing we can 4236 // do here and the operand rules may not line up with the below, so 4237 // bail early. 4238 if (Opcode >= ISD::BUILTIN_OP_END) 4239 return SDValue(); 4240 4241 if (isUndef(Opcode, {SDValue(Cst1, 0), SDValue(Cst2, 0)})) 4242 return getUNDEF(VT); 4243 4244 // Handle the case of two scalars. 4245 if (const ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1)) { 4246 if (const ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2)) { 4247 SDValue Folded = FoldConstantArithmetic(Opcode, DL, VT, Scalar1, Scalar2); 4248 assert((!Folded || !VT.isVector()) && 4249 "Can't fold vectors ops with scalar operands"); 4250 return Folded; 4251 } 4252 } 4253 4254 // fold (add Sym, c) -> Sym+c 4255 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Cst1)) 4256 return FoldSymbolOffset(Opcode, VT, GA, Cst2); 4257 if (TLI->isCommutativeBinOp(Opcode)) 4258 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Cst2)) 4259 return FoldSymbolOffset(Opcode, VT, GA, Cst1); 4260 4261 // For vectors extract each constant element into Inputs so we can constant 4262 // fold them individually. 4263 BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1); 4264 BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2); 4265 if (!BV1 || !BV2) 4266 return SDValue(); 4267 4268 assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!"); 4269 4270 EVT SVT = VT.getScalarType(); 4271 EVT LegalSVT = SVT; 4272 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) { 4273 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT); 4274 if (LegalSVT.bitsLT(SVT)) 4275 return SDValue(); 4276 } 4277 SmallVector<SDValue, 4> Outputs; 4278 for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) { 4279 SDValue V1 = BV1->getOperand(I); 4280 SDValue V2 = BV2->getOperand(I); 4281 4282 if (SVT.isInteger()) { 4283 if (V1->getValueType(0).bitsGT(SVT)) 4284 V1 = getNode(ISD::TRUNCATE, DL, SVT, V1); 4285 if (V2->getValueType(0).bitsGT(SVT)) 4286 V2 = getNode(ISD::TRUNCATE, DL, SVT, V2); 4287 } 4288 4289 if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT) 4290 return SDValue(); 4291 4292 // Fold one vector element. 4293 SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2); 4294 if (LegalSVT != SVT) 4295 ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult); 4296 4297 // Scalar folding only succeeded if the result is a constant or UNDEF. 4298 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant && 4299 ScalarResult.getOpcode() != ISD::ConstantFP) 4300 return SDValue(); 4301 Outputs.push_back(ScalarResult); 4302 } 4303 4304 assert(VT.getVectorNumElements() == Outputs.size() && 4305 "Vector size mismatch!"); 4306 4307 // We may have a vector type but a scalar result. Create a splat. 4308 Outputs.resize(VT.getVectorNumElements(), Outputs.back()); 4309 4310 // Build a big vector out of the scalar elements we generated. 4311 return getBuildVector(VT, SDLoc(), Outputs); 4312 } 4313 4314 // TODO: Merge with FoldConstantArithmetic 4315 SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode, 4316 const SDLoc &DL, EVT VT, 4317 ArrayRef<SDValue> Ops, 4318 const SDNodeFlags Flags) { 4319 // If the opcode is a target-specific ISD node, there's nothing we can 4320 // do here and the operand rules may not line up with the below, so 4321 // bail early. 4322 if (Opcode >= ISD::BUILTIN_OP_END) 4323 return SDValue(); 4324 4325 if (isUndef(Opcode, Ops)) 4326 return getUNDEF(VT); 4327 4328 // We can only fold vectors - maybe merge with FoldConstantArithmetic someday? 4329 if (!VT.isVector()) 4330 return SDValue(); 4331 4332 unsigned NumElts = VT.getVectorNumElements(); 4333 4334 auto IsScalarOrSameVectorSize = [&](const SDValue &Op) { 4335 return !Op.getValueType().isVector() || 4336 Op.getValueType().getVectorNumElements() == NumElts; 4337 }; 4338 4339 auto IsConstantBuildVectorOrUndef = [&](const SDValue &Op) { 4340 BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op); 4341 return (Op.isUndef()) || (Op.getOpcode() == ISD::CONDCODE) || 4342 (BV && BV->isConstant()); 4343 }; 4344 4345 // All operands must be vector types with the same number of elements as 4346 // the result type and must be either UNDEF or a build vector of constant 4347 // or UNDEF scalars. 4348 if (!llvm::all_of(Ops, IsConstantBuildVectorOrUndef) || 4349 !llvm::all_of(Ops, IsScalarOrSameVectorSize)) 4350 return SDValue(); 4351 4352 // If we are comparing vectors, then the result needs to be a i1 boolean 4353 // that is then sign-extended back to the legal result type. 4354 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType()); 4355 4356 // Find legal integer scalar type for constant promotion and 4357 // ensure that its scalar size is at least as large as source. 4358 EVT LegalSVT = VT.getScalarType(); 4359 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) { 4360 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT); 4361 if (LegalSVT.bitsLT(VT.getScalarType())) 4362 return SDValue(); 4363 } 4364 4365 // Constant fold each scalar lane separately. 4366 SmallVector<SDValue, 4> ScalarResults; 4367 for (unsigned i = 0; i != NumElts; i++) { 4368 SmallVector<SDValue, 4> ScalarOps; 4369 for (SDValue Op : Ops) { 4370 EVT InSVT = Op.getValueType().getScalarType(); 4371 BuildVectorSDNode *InBV = dyn_cast<BuildVectorSDNode>(Op); 4372 if (!InBV) { 4373 // We've checked that this is UNDEF or a constant of some kind. 4374 if (Op.isUndef()) 4375 ScalarOps.push_back(getUNDEF(InSVT)); 4376 else 4377 ScalarOps.push_back(Op); 4378 continue; 4379 } 4380 4381 SDValue ScalarOp = InBV->getOperand(i); 4382 EVT ScalarVT = ScalarOp.getValueType(); 4383 4384 // Build vector (integer) scalar operands may need implicit 4385 // truncation - do this before constant folding. 4386 if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT)) 4387 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp); 4388 4389 ScalarOps.push_back(ScalarOp); 4390 } 4391 4392 // Constant fold the scalar operands. 4393 SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags); 4394 4395 // Legalize the (integer) scalar constant if necessary. 4396 if (LegalSVT != SVT) 4397 ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult); 4398 4399 // Scalar folding only succeeded if the result is a constant or UNDEF. 4400 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant && 4401 ScalarResult.getOpcode() != ISD::ConstantFP) 4402 return SDValue(); 4403 ScalarResults.push_back(ScalarResult); 4404 } 4405 4406 SDValue V = getBuildVector(VT, DL, ScalarResults); 4407 NewSDValueDbgMsg(V, "New node fold constant vector: ", this); 4408 return V; 4409 } 4410 4411 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 4412 SDValue N1, SDValue N2, const SDNodeFlags Flags) { 4413 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 4414 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2); 4415 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 4416 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2); 4417 4418 // Canonicalize constant to RHS if commutative. 4419 if (TLI->isCommutativeBinOp(Opcode)) { 4420 if (N1C && !N2C) { 4421 std::swap(N1C, N2C); 4422 std::swap(N1, N2); 4423 } else if (N1CFP && !N2CFP) { 4424 std::swap(N1CFP, N2CFP); 4425 std::swap(N1, N2); 4426 } 4427 } 4428 4429 switch (Opcode) { 4430 default: break; 4431 case ISD::TokenFactor: 4432 assert(VT == MVT::Other && N1.getValueType() == MVT::Other && 4433 N2.getValueType() == MVT::Other && "Invalid token factor!"); 4434 // Fold trivial token factors. 4435 if (N1.getOpcode() == ISD::EntryToken) return N2; 4436 if (N2.getOpcode() == ISD::EntryToken) return N1; 4437 if (N1 == N2) return N1; 4438 break; 4439 case ISD::CONCAT_VECTORS: { 4440 // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF. 4441 SDValue Ops[] = {N1, N2}; 4442 if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this)) 4443 return V; 4444 break; 4445 } 4446 case ISD::AND: 4447 assert(VT.isInteger() && "This operator does not apply to FP types!"); 4448 assert(N1.getValueType() == N2.getValueType() && 4449 N1.getValueType() == VT && "Binary operator types must match!"); 4450 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's 4451 // worth handling here. 4452 if (N2C && N2C->isNullValue()) 4453 return N2; 4454 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X 4455 return N1; 4456 break; 4457 case ISD::OR: 4458 case ISD::XOR: 4459 case ISD::ADD: 4460 case ISD::SUB: 4461 assert(VT.isInteger() && "This operator does not apply to FP types!"); 4462 assert(N1.getValueType() == N2.getValueType() && 4463 N1.getValueType() == VT && "Binary operator types must match!"); 4464 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so 4465 // it's worth handling here. 4466 if (N2C && N2C->isNullValue()) 4467 return N1; 4468 break; 4469 case ISD::UDIV: 4470 case ISD::UREM: 4471 case ISD::MULHU: 4472 case ISD::MULHS: 4473 case ISD::MUL: 4474 case ISD::SDIV: 4475 case ISD::SREM: 4476 case ISD::SMIN: 4477 case ISD::SMAX: 4478 case ISD::UMIN: 4479 case ISD::UMAX: 4480 assert(VT.isInteger() && "This operator does not apply to FP types!"); 4481 assert(N1.getValueType() == N2.getValueType() && 4482 N1.getValueType() == VT && "Binary operator types must match!"); 4483 break; 4484 case ISD::FADD: 4485 case ISD::FSUB: 4486 case ISD::FMUL: 4487 case ISD::FDIV: 4488 case ISD::FREM: 4489 assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); 4490 assert(N1.getValueType() == N2.getValueType() && 4491 N1.getValueType() == VT && "Binary operator types must match!"); 4492 break; 4493 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. 4494 assert(N1.getValueType() == VT && 4495 N1.getValueType().isFloatingPoint() && 4496 N2.getValueType().isFloatingPoint() && 4497 "Invalid FCOPYSIGN!"); 4498 break; 4499 case ISD::SHL: 4500 case ISD::SRA: 4501 case ISD::SRL: 4502 case ISD::ROTL: 4503 case ISD::ROTR: 4504 assert(VT == N1.getValueType() && 4505 "Shift operators return type must be the same as their first arg"); 4506 assert(VT.isInteger() && N2.getValueType().isInteger() && 4507 "Shifts only work on integers"); 4508 assert((!VT.isVector() || VT == N2.getValueType()) && 4509 "Vector shift amounts must be in the same as their first arg"); 4510 // Verify that the shift amount VT is bit enough to hold valid shift 4511 // amounts. This catches things like trying to shift an i1024 value by an 4512 // i8, which is easy to fall into in generic code that uses 4513 // TLI.getShiftAmount(). 4514 assert(N2.getValueSizeInBits() >= Log2_32_Ceil(N1.getValueSizeInBits()) && 4515 "Invalid use of small shift amount with oversized value!"); 4516 4517 // Always fold shifts of i1 values so the code generator doesn't need to 4518 // handle them. Since we know the size of the shift has to be less than the 4519 // size of the value, the shift/rotate count is guaranteed to be zero. 4520 if (VT == MVT::i1) 4521 return N1; 4522 if (N2C && N2C->isNullValue()) 4523 return N1; 4524 break; 4525 case ISD::FP_ROUND_INREG: { 4526 EVT EVT = cast<VTSDNode>(N2)->getVT(); 4527 assert(VT == N1.getValueType() && "Not an inreg round!"); 4528 assert(VT.isFloatingPoint() && EVT.isFloatingPoint() && 4529 "Cannot FP_ROUND_INREG integer types"); 4530 assert(EVT.isVector() == VT.isVector() && 4531 "FP_ROUND_INREG type should be vector iff the operand " 4532 "type is vector!"); 4533 assert((!EVT.isVector() || 4534 EVT.getVectorNumElements() == VT.getVectorNumElements()) && 4535 "Vector element counts must match in FP_ROUND_INREG"); 4536 assert(EVT.bitsLE(VT) && "Not rounding down!"); 4537 (void)EVT; 4538 if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding. 4539 break; 4540 } 4541 case ISD::FP_ROUND: 4542 assert(VT.isFloatingPoint() && 4543 N1.getValueType().isFloatingPoint() && 4544 VT.bitsLE(N1.getValueType()) && 4545 N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && 4546 "Invalid FP_ROUND!"); 4547 if (N1.getValueType() == VT) return N1; // noop conversion. 4548 break; 4549 case ISD::AssertSext: 4550 case ISD::AssertZext: { 4551 EVT EVT = cast<VTSDNode>(N2)->getVT(); 4552 assert(VT == N1.getValueType() && "Not an inreg extend!"); 4553 assert(VT.isInteger() && EVT.isInteger() && 4554 "Cannot *_EXTEND_INREG FP types"); 4555 assert(!EVT.isVector() && 4556 "AssertSExt/AssertZExt type should be the vector element type " 4557 "rather than the vector type!"); 4558 assert(EVT.bitsLE(VT) && "Not extending!"); 4559 if (VT == EVT) return N1; // noop assertion. 4560 break; 4561 } 4562 case ISD::SIGN_EXTEND_INREG: { 4563 EVT EVT = cast<VTSDNode>(N2)->getVT(); 4564 assert(VT == N1.getValueType() && "Not an inreg extend!"); 4565 assert(VT.isInteger() && EVT.isInteger() && 4566 "Cannot *_EXTEND_INREG FP types"); 4567 assert(EVT.isVector() == VT.isVector() && 4568 "SIGN_EXTEND_INREG type should be vector iff the operand " 4569 "type is vector!"); 4570 assert((!EVT.isVector() || 4571 EVT.getVectorNumElements() == VT.getVectorNumElements()) && 4572 "Vector element counts must match in SIGN_EXTEND_INREG"); 4573 assert(EVT.bitsLE(VT) && "Not extending!"); 4574 if (EVT == VT) return N1; // Not actually extending 4575 4576 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) { 4577 unsigned FromBits = EVT.getScalarSizeInBits(); 4578 Val <<= Val.getBitWidth() - FromBits; 4579 Val.ashrInPlace(Val.getBitWidth() - FromBits); 4580 return getConstant(Val, DL, ConstantVT); 4581 }; 4582 4583 if (N1C) { 4584 const APInt &Val = N1C->getAPIntValue(); 4585 return SignExtendInReg(Val, VT); 4586 } 4587 if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) { 4588 SmallVector<SDValue, 8> Ops; 4589 llvm::EVT OpVT = N1.getOperand(0).getValueType(); 4590 for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) { 4591 SDValue Op = N1.getOperand(i); 4592 if (Op.isUndef()) { 4593 Ops.push_back(getUNDEF(OpVT)); 4594 continue; 4595 } 4596 ConstantSDNode *C = cast<ConstantSDNode>(Op); 4597 APInt Val = C->getAPIntValue(); 4598 Ops.push_back(SignExtendInReg(Val, OpVT)); 4599 } 4600 return getBuildVector(VT, DL, Ops); 4601 } 4602 break; 4603 } 4604 case ISD::EXTRACT_VECTOR_ELT: 4605 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() && 4606 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \ 4607 element type of the vector."); 4608 4609 // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF. 4610 if (N1.isUndef()) 4611 return getUNDEF(VT); 4612 4613 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF 4614 if (N2C && N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements())) 4615 return getUNDEF(VT); 4616 4617 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is 4618 // expanding copies of large vectors from registers. 4619 if (N2C && 4620 N1.getOpcode() == ISD::CONCAT_VECTORS && 4621 N1.getNumOperands() > 0) { 4622 unsigned Factor = 4623 N1.getOperand(0).getValueType().getVectorNumElements(); 4624 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, 4625 N1.getOperand(N2C->getZExtValue() / Factor), 4626 getConstant(N2C->getZExtValue() % Factor, DL, 4627 N2.getValueType())); 4628 } 4629 4630 // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is 4631 // expanding large vector constants. 4632 if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) { 4633 SDValue Elt = N1.getOperand(N2C->getZExtValue()); 4634 4635 if (VT != Elt.getValueType()) 4636 // If the vector element type is not legal, the BUILD_VECTOR operands 4637 // are promoted and implicitly truncated, and the result implicitly 4638 // extended. Make that explicit here. 4639 Elt = getAnyExtOrTrunc(Elt, DL, VT); 4640 4641 return Elt; 4642 } 4643 4644 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector 4645 // operations are lowered to scalars. 4646 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { 4647 // If the indices are the same, return the inserted element else 4648 // if the indices are known different, extract the element from 4649 // the original vector. 4650 SDValue N1Op2 = N1.getOperand(2); 4651 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2); 4652 4653 if (N1Op2C && N2C) { 4654 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { 4655 if (VT == N1.getOperand(1).getValueType()) 4656 return N1.getOperand(1); 4657 else 4658 return getSExtOrTrunc(N1.getOperand(1), DL, VT); 4659 } 4660 4661 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); 4662 } 4663 } 4664 4665 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed 4666 // when vector types are scalarized and v1iX is legal. 4667 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx) 4668 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && 4669 N1.getValueType().getVectorNumElements() == 1) { 4670 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), 4671 N1.getOperand(1)); 4672 } 4673 break; 4674 case ISD::EXTRACT_ELEMENT: 4675 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); 4676 assert(!N1.getValueType().isVector() && !VT.isVector() && 4677 (N1.getValueType().isInteger() == VT.isInteger()) && 4678 N1.getValueType() != VT && 4679 "Wrong types for EXTRACT_ELEMENT!"); 4680 4681 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding 4682 // 64-bit integers into 32-bit parts. Instead of building the extract of 4683 // the BUILD_PAIR, only to have legalize rip it apart, just do it now. 4684 if (N1.getOpcode() == ISD::BUILD_PAIR) 4685 return N1.getOperand(N2C->getZExtValue()); 4686 4687 // EXTRACT_ELEMENT of a constant int is also very common. 4688 if (N1C) { 4689 unsigned ElementSize = VT.getSizeInBits(); 4690 unsigned Shift = ElementSize * N2C->getZExtValue(); 4691 APInt ShiftedVal = N1C->getAPIntValue().lshr(Shift); 4692 return getConstant(ShiftedVal.trunc(ElementSize), DL, VT); 4693 } 4694 break; 4695 case ISD::EXTRACT_SUBVECTOR: 4696 if (VT.isSimple() && N1.getValueType().isSimple()) { 4697 assert(VT.isVector() && N1.getValueType().isVector() && 4698 "Extract subvector VTs must be a vectors!"); 4699 assert(VT.getVectorElementType() == 4700 N1.getValueType().getVectorElementType() && 4701 "Extract subvector VTs must have the same element type!"); 4702 assert(VT.getSimpleVT() <= N1.getSimpleValueType() && 4703 "Extract subvector must be from larger vector to smaller vector!"); 4704 4705 if (N2C) { 4706 assert((VT.getVectorNumElements() + N2C->getZExtValue() 4707 <= N1.getValueType().getVectorNumElements()) 4708 && "Extract subvector overflow!"); 4709 } 4710 4711 // Trivial extraction. 4712 if (VT.getSimpleVT() == N1.getSimpleValueType()) 4713 return N1; 4714 4715 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF. 4716 if (N1.isUndef()) 4717 return getUNDEF(VT); 4718 4719 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of 4720 // the concat have the same type as the extract. 4721 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS && 4722 N1.getNumOperands() > 0 && 4723 VT == N1.getOperand(0).getValueType()) { 4724 unsigned Factor = VT.getVectorNumElements(); 4725 return N1.getOperand(N2C->getZExtValue() / Factor); 4726 } 4727 4728 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created 4729 // during shuffle legalization. 4730 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) && 4731 VT == N1.getOperand(1).getValueType()) 4732 return N1.getOperand(1); 4733 } 4734 break; 4735 } 4736 4737 // Perform trivial constant folding. 4738 if (SDValue SV = 4739 FoldConstantArithmetic(Opcode, DL, VT, N1.getNode(), N2.getNode())) 4740 return SV; 4741 4742 // Constant fold FP operations. 4743 bool HasFPExceptions = TLI->hasFloatingPointExceptions(); 4744 if (N1CFP) { 4745 if (N2CFP) { 4746 APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF(); 4747 APFloat::opStatus s; 4748 switch (Opcode) { 4749 case ISD::FADD: 4750 s = V1.add(V2, APFloat::rmNearestTiesToEven); 4751 if (!HasFPExceptions || s != APFloat::opInvalidOp) 4752 return getConstantFP(V1, DL, VT); 4753 break; 4754 case ISD::FSUB: 4755 s = V1.subtract(V2, APFloat::rmNearestTiesToEven); 4756 if (!HasFPExceptions || s!=APFloat::opInvalidOp) 4757 return getConstantFP(V1, DL, VT); 4758 break; 4759 case ISD::FMUL: 4760 s = V1.multiply(V2, APFloat::rmNearestTiesToEven); 4761 if (!HasFPExceptions || s!=APFloat::opInvalidOp) 4762 return getConstantFP(V1, DL, VT); 4763 break; 4764 case ISD::FDIV: 4765 s = V1.divide(V2, APFloat::rmNearestTiesToEven); 4766 if (!HasFPExceptions || (s!=APFloat::opInvalidOp && 4767 s!=APFloat::opDivByZero)) { 4768 return getConstantFP(V1, DL, VT); 4769 } 4770 break; 4771 case ISD::FREM : 4772 s = V1.mod(V2); 4773 if (!HasFPExceptions || (s!=APFloat::opInvalidOp && 4774 s!=APFloat::opDivByZero)) { 4775 return getConstantFP(V1, DL, VT); 4776 } 4777 break; 4778 case ISD::FCOPYSIGN: 4779 V1.copySign(V2); 4780 return getConstantFP(V1, DL, VT); 4781 default: break; 4782 } 4783 } 4784 4785 if (Opcode == ISD::FP_ROUND) { 4786 APFloat V = N1CFP->getValueAPF(); // make copy 4787 bool ignored; 4788 // This can return overflow, underflow, or inexact; we don't care. 4789 // FIXME need to be more flexible about rounding mode. 4790 (void)V.convert(EVTToAPFloatSemantics(VT), 4791 APFloat::rmNearestTiesToEven, &ignored); 4792 return getConstantFP(V, DL, VT); 4793 } 4794 } 4795 4796 // Any FP binop with an undef operand is folded to NaN. This matches the 4797 // behavior of the IR optimizer. 4798 switch (Opcode) { 4799 case ISD::FADD: 4800 case ISD::FSUB: 4801 case ISD::FMUL: 4802 case ISD::FDIV: 4803 case ISD::FREM: 4804 if (N1.isUndef() || N2.isUndef()) 4805 return getConstantFP(APFloat::getNaN(EVTToAPFloatSemantics(VT)), DL, VT); 4806 } 4807 4808 // Canonicalize an UNDEF to the RHS, even over a constant. 4809 if (N1.isUndef()) { 4810 if (TLI->isCommutativeBinOp(Opcode)) { 4811 std::swap(N1, N2); 4812 } else { 4813 switch (Opcode) { 4814 case ISD::FP_ROUND_INREG: 4815 case ISD::SIGN_EXTEND_INREG: 4816 case ISD::SUB: 4817 return getUNDEF(VT); // fold op(undef, arg2) -> undef 4818 case ISD::UDIV: 4819 case ISD::SDIV: 4820 case ISD::UREM: 4821 case ISD::SREM: 4822 case ISD::SRA: 4823 case ISD::SRL: 4824 case ISD::SHL: 4825 return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0 4826 } 4827 } 4828 } 4829 4830 // Fold a bunch of operators when the RHS is undef. 4831 if (N2.isUndef()) { 4832 switch (Opcode) { 4833 case ISD::XOR: 4834 if (N1.isUndef()) 4835 // Handle undef ^ undef -> 0 special case. This is a common 4836 // idiom (misuse). 4837 return getConstant(0, DL, VT); 4838 LLVM_FALLTHROUGH; 4839 case ISD::ADD: 4840 case ISD::ADDC: 4841 case ISD::ADDE: 4842 case ISD::SUB: 4843 case ISD::UDIV: 4844 case ISD::SDIV: 4845 case ISD::UREM: 4846 case ISD::SREM: 4847 case ISD::SRA: 4848 case ISD::SRL: 4849 case ISD::SHL: 4850 return getUNDEF(VT); // fold op(arg1, undef) -> undef 4851 case ISD::MUL: 4852 case ISD::AND: 4853 return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0 4854 case ISD::OR: 4855 return getAllOnesConstant(DL, VT); 4856 } 4857 } 4858 4859 // Memoize this node if possible. 4860 SDNode *N; 4861 SDVTList VTs = getVTList(VT); 4862 SDValue Ops[] = {N1, N2}; 4863 if (VT != MVT::Glue) { 4864 FoldingSetNodeID ID; 4865 AddNodeIDNode(ID, Opcode, VTs, Ops); 4866 void *IP = nullptr; 4867 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { 4868 E->intersectFlagsWith(Flags); 4869 return SDValue(E, 0); 4870 } 4871 4872 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 4873 N->setFlags(Flags); 4874 createOperands(N, Ops); 4875 CSEMap.InsertNode(N, IP); 4876 } else { 4877 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 4878 createOperands(N, Ops); 4879 } 4880 4881 InsertNode(N); 4882 SDValue V = SDValue(N, 0); 4883 NewSDValueDbgMsg(V, "Creating new node: ", this); 4884 return V; 4885 } 4886 4887 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 4888 SDValue N1, SDValue N2, SDValue N3, 4889 const SDNodeFlags Flags) { 4890 // Perform various simplifications. 4891 switch (Opcode) { 4892 case ISD::FMA: { 4893 assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); 4894 assert(N1.getValueType() == VT && N2.getValueType() == VT && 4895 N3.getValueType() == VT && "FMA types must match!"); 4896 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 4897 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2); 4898 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3); 4899 if (N1CFP && N2CFP && N3CFP) { 4900 APFloat V1 = N1CFP->getValueAPF(); 4901 const APFloat &V2 = N2CFP->getValueAPF(); 4902 const APFloat &V3 = N3CFP->getValueAPF(); 4903 APFloat::opStatus s = 4904 V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven); 4905 if (!TLI->hasFloatingPointExceptions() || s != APFloat::opInvalidOp) 4906 return getConstantFP(V1, DL, VT); 4907 } 4908 break; 4909 } 4910 case ISD::CONCAT_VECTORS: { 4911 // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF. 4912 SDValue Ops[] = {N1, N2, N3}; 4913 if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this)) 4914 return V; 4915 break; 4916 } 4917 case ISD::SETCC: { 4918 // Use FoldSetCC to simplify SETCC's. 4919 if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL)) 4920 return V; 4921 // Vector constant folding. 4922 SDValue Ops[] = {N1, N2, N3}; 4923 if (SDValue V = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) { 4924 NewSDValueDbgMsg(V, "New node vector constant folding: ", this); 4925 return V; 4926 } 4927 break; 4928 } 4929 case ISD::SELECT: 4930 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) { 4931 if (N1C->getZExtValue()) 4932 return N2; // select true, X, Y -> X 4933 return N3; // select false, X, Y -> Y 4934 } 4935 4936 if (N2 == N3) return N2; // select C, X, X -> X 4937 break; 4938 case ISD::VECTOR_SHUFFLE: 4939 llvm_unreachable("should use getVectorShuffle constructor!"); 4940 case ISD::INSERT_VECTOR_ELT: { 4941 ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3); 4942 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF 4943 if (N3C && N3C->getZExtValue() >= N1.getValueType().getVectorNumElements()) 4944 return getUNDEF(VT); 4945 break; 4946 } 4947 case ISD::INSERT_SUBVECTOR: { 4948 SDValue Index = N3; 4949 if (VT.isSimple() && N1.getValueType().isSimple() 4950 && N2.getValueType().isSimple()) { 4951 assert(VT.isVector() && N1.getValueType().isVector() && 4952 N2.getValueType().isVector() && 4953 "Insert subvector VTs must be a vectors"); 4954 assert(VT == N1.getValueType() && 4955 "Dest and insert subvector source types must match!"); 4956 assert(N2.getSimpleValueType() <= N1.getSimpleValueType() && 4957 "Insert subvector must be from smaller vector to larger vector!"); 4958 if (isa<ConstantSDNode>(Index)) { 4959 assert((N2.getValueType().getVectorNumElements() + 4960 cast<ConstantSDNode>(Index)->getZExtValue() 4961 <= VT.getVectorNumElements()) 4962 && "Insert subvector overflow!"); 4963 } 4964 4965 // Trivial insertion. 4966 if (VT.getSimpleVT() == N2.getSimpleValueType()) 4967 return N2; 4968 } 4969 break; 4970 } 4971 case ISD::BITCAST: 4972 // Fold bit_convert nodes from a type to themselves. 4973 if (N1.getValueType() == VT) 4974 return N1; 4975 break; 4976 } 4977 4978 // Memoize node if it doesn't produce a flag. 4979 SDNode *N; 4980 SDVTList VTs = getVTList(VT); 4981 SDValue Ops[] = {N1, N2, N3}; 4982 if (VT != MVT::Glue) { 4983 FoldingSetNodeID ID; 4984 AddNodeIDNode(ID, Opcode, VTs, Ops); 4985 void *IP = nullptr; 4986 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { 4987 E->intersectFlagsWith(Flags); 4988 return SDValue(E, 0); 4989 } 4990 4991 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 4992 N->setFlags(Flags); 4993 createOperands(N, Ops); 4994 CSEMap.InsertNode(N, IP); 4995 } else { 4996 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 4997 createOperands(N, Ops); 4998 } 4999 5000 InsertNode(N); 5001 SDValue V = SDValue(N, 0); 5002 NewSDValueDbgMsg(V, "Creating new node: ", this); 5003 return V; 5004 } 5005 5006 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 5007 SDValue N1, SDValue N2, SDValue N3, SDValue N4) { 5008 SDValue Ops[] = { N1, N2, N3, N4 }; 5009 return getNode(Opcode, DL, VT, Ops); 5010 } 5011 5012 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 5013 SDValue N1, SDValue N2, SDValue N3, SDValue N4, 5014 SDValue N5) { 5015 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 5016 return getNode(Opcode, DL, VT, Ops); 5017 } 5018 5019 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all 5020 /// the incoming stack arguments to be loaded from the stack. 5021 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { 5022 SmallVector<SDValue, 8> ArgChains; 5023 5024 // Include the original chain at the beginning of the list. When this is 5025 // used by target LowerCall hooks, this helps legalize find the 5026 // CALLSEQ_BEGIN node. 5027 ArgChains.push_back(Chain); 5028 5029 // Add a chain value for each stack argument. 5030 for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(), 5031 UE = getEntryNode().getNode()->use_end(); U != UE; ++U) 5032 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U)) 5033 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) 5034 if (FI->getIndex() < 0) 5035 ArgChains.push_back(SDValue(L, 1)); 5036 5037 // Build a tokenfactor for all the chains. 5038 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); 5039 } 5040 5041 /// getMemsetValue - Vectorized representation of the memset value 5042 /// operand. 5043 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, 5044 const SDLoc &dl) { 5045 assert(!Value.isUndef()); 5046 5047 unsigned NumBits = VT.getScalarSizeInBits(); 5048 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { 5049 assert(C->getAPIntValue().getBitWidth() == 8); 5050 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue()); 5051 if (VT.isInteger()) 5052 return DAG.getConstant(Val, dl, VT); 5053 return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl, 5054 VT); 5055 } 5056 5057 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?"); 5058 EVT IntVT = VT.getScalarType(); 5059 if (!IntVT.isInteger()) 5060 IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits()); 5061 5062 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value); 5063 if (NumBits > 8) { 5064 // Use a multiplication with 0x010101... to extend the input to the 5065 // required length. 5066 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01)); 5067 Value = DAG.getNode(ISD::MUL, dl, IntVT, Value, 5068 DAG.getConstant(Magic, dl, IntVT)); 5069 } 5070 5071 if (VT != Value.getValueType() && !VT.isInteger()) 5072 Value = DAG.getBitcast(VT.getScalarType(), Value); 5073 if (VT != Value.getValueType()) 5074 Value = DAG.getSplatBuildVector(VT, dl, Value); 5075 5076 return Value; 5077 } 5078 5079 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only 5080 /// used when a memcpy is turned into a memset when the source is a constant 5081 /// string ptr. 5082 static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, 5083 const TargetLowering &TLI, 5084 const ConstantDataArraySlice &Slice) { 5085 // Handle vector with all elements zero. 5086 if (Slice.Array == nullptr) { 5087 if (VT.isInteger()) 5088 return DAG.getConstant(0, dl, VT); 5089 else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128) 5090 return DAG.getConstantFP(0.0, dl, VT); 5091 else if (VT.isVector()) { 5092 unsigned NumElts = VT.getVectorNumElements(); 5093 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; 5094 return DAG.getNode(ISD::BITCAST, dl, VT, 5095 DAG.getConstant(0, dl, 5096 EVT::getVectorVT(*DAG.getContext(), 5097 EltVT, NumElts))); 5098 } else 5099 llvm_unreachable("Expected type!"); 5100 } 5101 5102 assert(!VT.isVector() && "Can't handle vector type here!"); 5103 unsigned NumVTBits = VT.getSizeInBits(); 5104 unsigned NumVTBytes = NumVTBits / 8; 5105 unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length)); 5106 5107 APInt Val(NumVTBits, 0); 5108 if (DAG.getDataLayout().isLittleEndian()) { 5109 for (unsigned i = 0; i != NumBytes; ++i) 5110 Val |= (uint64_t)(unsigned char)Slice[i] << i*8; 5111 } else { 5112 for (unsigned i = 0; i != NumBytes; ++i) 5113 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8; 5114 } 5115 5116 // If the "cost" of materializing the integer immediate is less than the cost 5117 // of a load, then it is cost effective to turn the load into the immediate. 5118 Type *Ty = VT.getTypeForEVT(*DAG.getContext()); 5119 if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty)) 5120 return DAG.getConstant(Val, dl, VT); 5121 return SDValue(nullptr, 0); 5122 } 5123 5124 SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, unsigned Offset, 5125 const SDLoc &DL) { 5126 EVT VT = Base.getValueType(); 5127 return getNode(ISD::ADD, DL, VT, Base, getConstant(Offset, DL, VT)); 5128 } 5129 5130 /// Returns true if memcpy source is constant data. 5131 static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) { 5132 uint64_t SrcDelta = 0; 5133 GlobalAddressSDNode *G = nullptr; 5134 if (Src.getOpcode() == ISD::GlobalAddress) 5135 G = cast<GlobalAddressSDNode>(Src); 5136 else if (Src.getOpcode() == ISD::ADD && 5137 Src.getOperand(0).getOpcode() == ISD::GlobalAddress && 5138 Src.getOperand(1).getOpcode() == ISD::Constant) { 5139 G = cast<GlobalAddressSDNode>(Src.getOperand(0)); 5140 SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue(); 5141 } 5142 if (!G) 5143 return false; 5144 5145 return getConstantDataArrayInfo(G->getGlobal(), Slice, 8, 5146 SrcDelta + G->getOffset()); 5147 } 5148 5149 /// Determines the optimal series of memory ops to replace the memset / memcpy. 5150 /// Return true if the number of memory ops is below the threshold (Limit). 5151 /// It returns the types of the sequence of memory ops to perform 5152 /// memset / memcpy by reference. 5153 static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, 5154 unsigned Limit, uint64_t Size, 5155 unsigned DstAlign, unsigned SrcAlign, 5156 bool IsMemset, 5157 bool ZeroMemset, 5158 bool MemcpyStrSrc, 5159 bool AllowOverlap, 5160 unsigned DstAS, unsigned SrcAS, 5161 SelectionDAG &DAG, 5162 const TargetLowering &TLI) { 5163 assert((SrcAlign == 0 || SrcAlign >= DstAlign) && 5164 "Expecting memcpy / memset source to meet alignment requirement!"); 5165 // If 'SrcAlign' is zero, that means the memory operation does not need to 5166 // load the value, i.e. memset or memcpy from constant string. Otherwise, 5167 // it's the inferred alignment of the source. 'DstAlign', on the other hand, 5168 // is the specified alignment of the memory operation. If it is zero, that 5169 // means it's possible to change the alignment of the destination. 5170 // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does 5171 // not need to be loaded. 5172 EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, 5173 IsMemset, ZeroMemset, MemcpyStrSrc, 5174 DAG.getMachineFunction()); 5175 5176 if (VT == MVT::Other) { 5177 // Use the largest integer type whose alignment constraints are satisfied. 5178 // We only need to check DstAlign here as SrcAlign is always greater or 5179 // equal to DstAlign (or zero). 5180 VT = MVT::i64; 5181 while (DstAlign && DstAlign < VT.getSizeInBits() / 8 && 5182 !TLI.allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign)) 5183 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); 5184 assert(VT.isInteger()); 5185 5186 // Find the largest legal integer type. 5187 MVT LVT = MVT::i64; 5188 while (!TLI.isTypeLegal(LVT)) 5189 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1); 5190 assert(LVT.isInteger()); 5191 5192 // If the type we've chosen is larger than the largest legal integer type 5193 // then use that instead. 5194 if (VT.bitsGT(LVT)) 5195 VT = LVT; 5196 } 5197 5198 unsigned NumMemOps = 0; 5199 while (Size != 0) { 5200 unsigned VTSize = VT.getSizeInBits() / 8; 5201 while (VTSize > Size) { 5202 // For now, only use non-vector load / store's for the left-over pieces. 5203 EVT NewVT = VT; 5204 unsigned NewVTSize; 5205 5206 bool Found = false; 5207 if (VT.isVector() || VT.isFloatingPoint()) { 5208 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32; 5209 if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) && 5210 TLI.isSafeMemOpType(NewVT.getSimpleVT())) 5211 Found = true; 5212 else if (NewVT == MVT::i64 && 5213 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) && 5214 TLI.isSafeMemOpType(MVT::f64)) { 5215 // i64 is usually not legal on 32-bit targets, but f64 may be. 5216 NewVT = MVT::f64; 5217 Found = true; 5218 } 5219 } 5220 5221 if (!Found) { 5222 do { 5223 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1); 5224 if (NewVT == MVT::i8) 5225 break; 5226 } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT())); 5227 } 5228 NewVTSize = NewVT.getSizeInBits() / 8; 5229 5230 // If the new VT cannot cover all of the remaining bits, then consider 5231 // issuing a (or a pair of) unaligned and overlapping load / store. 5232 // FIXME: Only does this for 64-bit or more since we don't have proper 5233 // cost model for unaligned load / store. 5234 bool Fast; 5235 if (NumMemOps && AllowOverlap && 5236 VTSize >= 8 && NewVTSize < Size && 5237 TLI.allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign, &Fast) && Fast) 5238 VTSize = Size; 5239 else { 5240 VT = NewVT; 5241 VTSize = NewVTSize; 5242 } 5243 } 5244 5245 if (++NumMemOps > Limit) 5246 return false; 5247 5248 MemOps.push_back(VT); 5249 Size -= VTSize; 5250 } 5251 5252 return true; 5253 } 5254 5255 static bool shouldLowerMemFuncForSize(const MachineFunction &MF) { 5256 // On Darwin, -Os means optimize for size without hurting performance, so 5257 // only really optimize for size when -Oz (MinSize) is used. 5258 if (MF.getTarget().getTargetTriple().isOSDarwin()) 5259 return MF.getFunction().optForMinSize(); 5260 return MF.getFunction().optForSize(); 5261 } 5262 5263 static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl, 5264 SmallVector<SDValue, 32> &OutChains, unsigned From, 5265 unsigned To, SmallVector<SDValue, 16> &OutLoadChains, 5266 SmallVector<SDValue, 16> &OutStoreChains) { 5267 assert(OutLoadChains.size() && "Missing loads in memcpy inlining"); 5268 assert(OutStoreChains.size() && "Missing stores in memcpy inlining"); 5269 SmallVector<SDValue, 16> GluedLoadChains; 5270 for (unsigned i = From; i < To; ++i) { 5271 OutChains.push_back(OutLoadChains[i]); 5272 GluedLoadChains.push_back(OutLoadChains[i]); 5273 } 5274 5275 // Chain for all loads. 5276 SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 5277 GluedLoadChains); 5278 5279 for (unsigned i = From; i < To; ++i) { 5280 StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]); 5281 SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(), 5282 ST->getBasePtr(), ST->getMemoryVT(), 5283 ST->getMemOperand()); 5284 OutChains.push_back(NewStore); 5285 } 5286 } 5287 5288 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, 5289 SDValue Chain, SDValue Dst, SDValue Src, 5290 uint64_t Size, unsigned Align, 5291 bool isVol, bool AlwaysInline, 5292 MachinePointerInfo DstPtrInfo, 5293 MachinePointerInfo SrcPtrInfo) { 5294 // Turn a memcpy of undef to nop. 5295 if (Src.isUndef()) 5296 return Chain; 5297 5298 // Expand memcpy to a series of load and store ops if the size operand falls 5299 // below a certain threshold. 5300 // TODO: In the AlwaysInline case, if the size is big then generate a loop 5301 // rather than maybe a humongous number of loads and stores. 5302 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 5303 const DataLayout &DL = DAG.getDataLayout(); 5304 LLVMContext &C = *DAG.getContext(); 5305 std::vector<EVT> MemOps; 5306 bool DstAlignCanChange = false; 5307 MachineFunction &MF = DAG.getMachineFunction(); 5308 MachineFrameInfo &MFI = MF.getFrameInfo(); 5309 bool OptSize = shouldLowerMemFuncForSize(MF); 5310 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 5311 if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) 5312 DstAlignCanChange = true; 5313 unsigned SrcAlign = DAG.InferPtrAlignment(Src); 5314 if (Align > SrcAlign) 5315 SrcAlign = Align; 5316 ConstantDataArraySlice Slice; 5317 bool CopyFromConstant = isMemSrcFromConstant(Src, Slice); 5318 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr; 5319 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); 5320 5321 if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 5322 (DstAlignCanChange ? 0 : Align), 5323 (isZeroConstant ? 0 : SrcAlign), 5324 false, false, CopyFromConstant, true, 5325 DstPtrInfo.getAddrSpace(), 5326 SrcPtrInfo.getAddrSpace(), 5327 DAG, TLI)) 5328 return SDValue(); 5329 5330 if (DstAlignCanChange) { 5331 Type *Ty = MemOps[0].getTypeForEVT(C); 5332 unsigned NewAlign = (unsigned)DL.getABITypeAlignment(Ty); 5333 5334 // Don't promote to an alignment that would require dynamic stack 5335 // realignment. 5336 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); 5337 if (!TRI->needsStackRealignment(MF)) 5338 while (NewAlign > Align && 5339 DL.exceedsNaturalStackAlignment(NewAlign)) 5340 NewAlign /= 2; 5341 5342 if (NewAlign > Align) { 5343 // Give the stack frame object a larger alignment if needed. 5344 if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign) 5345 MFI.setObjectAlignment(FI->getIndex(), NewAlign); 5346 Align = NewAlign; 5347 } 5348 } 5349 5350 MachineMemOperand::Flags MMOFlags = 5351 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; 5352 SmallVector<SDValue, 16> OutLoadChains; 5353 SmallVector<SDValue, 16> OutStoreChains; 5354 SmallVector<SDValue, 32> OutChains; 5355 unsigned NumMemOps = MemOps.size(); 5356 uint64_t SrcOff = 0, DstOff = 0; 5357 for (unsigned i = 0; i != NumMemOps; ++i) { 5358 EVT VT = MemOps[i]; 5359 unsigned VTSize = VT.getSizeInBits() / 8; 5360 SDValue Value, Store; 5361 5362 if (VTSize > Size) { 5363 // Issuing an unaligned load / store pair that overlaps with the previous 5364 // pair. Adjust the offset accordingly. 5365 assert(i == NumMemOps-1 && i != 0); 5366 SrcOff -= VTSize - Size; 5367 DstOff -= VTSize - Size; 5368 } 5369 5370 if (CopyFromConstant && 5371 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) { 5372 // It's unlikely a store of a vector immediate can be done in a single 5373 // instruction. It would require a load from a constantpool first. 5374 // We only handle zero vectors here. 5375 // FIXME: Handle other cases where store of vector immediate is done in 5376 // a single instruction. 5377 ConstantDataArraySlice SubSlice; 5378 if (SrcOff < Slice.Length) { 5379 SubSlice = Slice; 5380 SubSlice.move(SrcOff); 5381 } else { 5382 // This is an out-of-bounds access and hence UB. Pretend we read zero. 5383 SubSlice.Array = nullptr; 5384 SubSlice.Offset = 0; 5385 SubSlice.Length = VTSize; 5386 } 5387 Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice); 5388 if (Value.getNode()) { 5389 Store = DAG.getStore(Chain, dl, Value, 5390 DAG.getMemBasePlusOffset(Dst, DstOff, dl), 5391 DstPtrInfo.getWithOffset(DstOff), Align, 5392 MMOFlags); 5393 OutChains.push_back(Store); 5394 } 5395 } 5396 5397 if (!Store.getNode()) { 5398 // The type might not be legal for the target. This should only happen 5399 // if the type is smaller than a legal type, as on PPC, so the right 5400 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify 5401 // to Load/Store if NVT==VT. 5402 // FIXME does the case above also need this? 5403 EVT NVT = TLI.getTypeToTransformTo(C, VT); 5404 assert(NVT.bitsGE(VT)); 5405 5406 bool isDereferenceable = 5407 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL); 5408 MachineMemOperand::Flags SrcMMOFlags = MMOFlags; 5409 if (isDereferenceable) 5410 SrcMMOFlags |= MachineMemOperand::MODereferenceable; 5411 5412 Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain, 5413 DAG.getMemBasePlusOffset(Src, SrcOff, dl), 5414 SrcPtrInfo.getWithOffset(SrcOff), VT, 5415 MinAlign(SrcAlign, SrcOff), SrcMMOFlags); 5416 OutLoadChains.push_back(Value.getValue(1)); 5417 5418 Store = DAG.getTruncStore( 5419 Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl), 5420 DstPtrInfo.getWithOffset(DstOff), VT, Align, MMOFlags); 5421 OutStoreChains.push_back(Store); 5422 } 5423 SrcOff += VTSize; 5424 DstOff += VTSize; 5425 Size -= VTSize; 5426 } 5427 5428 unsigned GluedLdStLimit = MaxLdStGlue == 0 ? 5429 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue; 5430 unsigned NumLdStInMemcpy = OutStoreChains.size(); 5431 5432 if (NumLdStInMemcpy) { 5433 // It may be that memcpy might be converted to memset if it's memcpy 5434 // of constants. In such a case, we won't have loads and stores, but 5435 // just stores. In the absence of loads, there is nothing to gang up. 5436 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) { 5437 // If target does not care, just leave as it. 5438 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) { 5439 OutChains.push_back(OutLoadChains[i]); 5440 OutChains.push_back(OutStoreChains[i]); 5441 } 5442 } else { 5443 // Ld/St less than/equal limit set by target. 5444 if (NumLdStInMemcpy <= GluedLdStLimit) { 5445 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0, 5446 NumLdStInMemcpy, OutLoadChains, 5447 OutStoreChains); 5448 } else { 5449 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit; 5450 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit; 5451 unsigned GlueIter = 0; 5452 5453 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) { 5454 unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit; 5455 unsigned IndexTo = NumLdStInMemcpy - GlueIter; 5456 5457 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo, 5458 OutLoadChains, OutStoreChains); 5459 GlueIter += GluedLdStLimit; 5460 } 5461 5462 // Residual ld/st. 5463 if (RemainingLdStInMemcpy) { 5464 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0, 5465 RemainingLdStInMemcpy, OutLoadChains, 5466 OutStoreChains); 5467 } 5468 } 5469 } 5470 } 5471 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); 5472 } 5473 5474 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, 5475 SDValue Chain, SDValue Dst, SDValue Src, 5476 uint64_t Size, unsigned Align, 5477 bool isVol, bool AlwaysInline, 5478 MachinePointerInfo DstPtrInfo, 5479 MachinePointerInfo SrcPtrInfo) { 5480 // Turn a memmove of undef to nop. 5481 if (Src.isUndef()) 5482 return Chain; 5483 5484 // Expand memmove to a series of load and store ops if the size operand falls 5485 // below a certain threshold. 5486 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 5487 const DataLayout &DL = DAG.getDataLayout(); 5488 LLVMContext &C = *DAG.getContext(); 5489 std::vector<EVT> MemOps; 5490 bool DstAlignCanChange = false; 5491 MachineFunction &MF = DAG.getMachineFunction(); 5492 MachineFrameInfo &MFI = MF.getFrameInfo(); 5493 bool OptSize = shouldLowerMemFuncForSize(MF); 5494 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 5495 if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) 5496 DstAlignCanChange = true; 5497 unsigned SrcAlign = DAG.InferPtrAlignment(Src); 5498 if (Align > SrcAlign) 5499 SrcAlign = Align; 5500 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); 5501 5502 if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 5503 (DstAlignCanChange ? 0 : Align), SrcAlign, 5504 false, false, false, false, 5505 DstPtrInfo.getAddrSpace(), 5506 SrcPtrInfo.getAddrSpace(), 5507 DAG, TLI)) 5508 return SDValue(); 5509 5510 if (DstAlignCanChange) { 5511 Type *Ty = MemOps[0].getTypeForEVT(C); 5512 unsigned NewAlign = (unsigned)DL.getABITypeAlignment(Ty); 5513 if (NewAlign > Align) { 5514 // Give the stack frame object a larger alignment if needed. 5515 if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign) 5516 MFI.setObjectAlignment(FI->getIndex(), NewAlign); 5517 Align = NewAlign; 5518 } 5519 } 5520 5521 MachineMemOperand::Flags MMOFlags = 5522 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; 5523 uint64_t SrcOff = 0, DstOff = 0; 5524 SmallVector<SDValue, 8> LoadValues; 5525 SmallVector<SDValue, 8> LoadChains; 5526 SmallVector<SDValue, 8> OutChains; 5527 unsigned NumMemOps = MemOps.size(); 5528 for (unsigned i = 0; i < NumMemOps; i++) { 5529 EVT VT = MemOps[i]; 5530 unsigned VTSize = VT.getSizeInBits() / 8; 5531 SDValue Value; 5532 5533 bool isDereferenceable = 5534 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL); 5535 MachineMemOperand::Flags SrcMMOFlags = MMOFlags; 5536 if (isDereferenceable) 5537 SrcMMOFlags |= MachineMemOperand::MODereferenceable; 5538 5539 Value = 5540 DAG.getLoad(VT, dl, Chain, DAG.getMemBasePlusOffset(Src, SrcOff, dl), 5541 SrcPtrInfo.getWithOffset(SrcOff), SrcAlign, SrcMMOFlags); 5542 LoadValues.push_back(Value); 5543 LoadChains.push_back(Value.getValue(1)); 5544 SrcOff += VTSize; 5545 } 5546 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); 5547 OutChains.clear(); 5548 for (unsigned i = 0; i < NumMemOps; i++) { 5549 EVT VT = MemOps[i]; 5550 unsigned VTSize = VT.getSizeInBits() / 8; 5551 SDValue Store; 5552 5553 Store = DAG.getStore(Chain, dl, LoadValues[i], 5554 DAG.getMemBasePlusOffset(Dst, DstOff, dl), 5555 DstPtrInfo.getWithOffset(DstOff), Align, MMOFlags); 5556 OutChains.push_back(Store); 5557 DstOff += VTSize; 5558 } 5559 5560 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); 5561 } 5562 5563 /// Lower the call to 'memset' intrinsic function into a series of store 5564 /// operations. 5565 /// 5566 /// \param DAG Selection DAG where lowered code is placed. 5567 /// \param dl Link to corresponding IR location. 5568 /// \param Chain Control flow dependency. 5569 /// \param Dst Pointer to destination memory location. 5570 /// \param Src Value of byte to write into the memory. 5571 /// \param Size Number of bytes to write. 5572 /// \param Align Alignment of the destination in bytes. 5573 /// \param isVol True if destination is volatile. 5574 /// \param DstPtrInfo IR information on the memory pointer. 5575 /// \returns New head in the control flow, if lowering was successful, empty 5576 /// SDValue otherwise. 5577 /// 5578 /// The function tries to replace 'llvm.memset' intrinsic with several store 5579 /// operations and value calculation code. This is usually profitable for small 5580 /// memory size. 5581 static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, 5582 SDValue Chain, SDValue Dst, SDValue Src, 5583 uint64_t Size, unsigned Align, bool isVol, 5584 MachinePointerInfo DstPtrInfo) { 5585 // Turn a memset of undef to nop. 5586 if (Src.isUndef()) 5587 return Chain; 5588 5589 // Expand memset to a series of load/store ops if the size operand 5590 // falls below a certain threshold. 5591 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 5592 std::vector<EVT> MemOps; 5593 bool DstAlignCanChange = false; 5594 MachineFunction &MF = DAG.getMachineFunction(); 5595 MachineFrameInfo &MFI = MF.getFrameInfo(); 5596 bool OptSize = shouldLowerMemFuncForSize(MF); 5597 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 5598 if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) 5599 DstAlignCanChange = true; 5600 bool IsZeroVal = 5601 isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); 5602 if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize), 5603 Size, (DstAlignCanChange ? 0 : Align), 0, 5604 true, IsZeroVal, false, true, 5605 DstPtrInfo.getAddrSpace(), ~0u, 5606 DAG, TLI)) 5607 return SDValue(); 5608 5609 if (DstAlignCanChange) { 5610 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 5611 unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty); 5612 if (NewAlign > Align) { 5613 // Give the stack frame object a larger alignment if needed. 5614 if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign) 5615 MFI.setObjectAlignment(FI->getIndex(), NewAlign); 5616 Align = NewAlign; 5617 } 5618 } 5619 5620 SmallVector<SDValue, 8> OutChains; 5621 uint64_t DstOff = 0; 5622 unsigned NumMemOps = MemOps.size(); 5623 5624 // Find the largest store and generate the bit pattern for it. 5625 EVT LargestVT = MemOps[0]; 5626 for (unsigned i = 1; i < NumMemOps; i++) 5627 if (MemOps[i].bitsGT(LargestVT)) 5628 LargestVT = MemOps[i]; 5629 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); 5630 5631 for (unsigned i = 0; i < NumMemOps; i++) { 5632 EVT VT = MemOps[i]; 5633 unsigned VTSize = VT.getSizeInBits() / 8; 5634 if (VTSize > Size) { 5635 // Issuing an unaligned load / store pair that overlaps with the previous 5636 // pair. Adjust the offset accordingly. 5637 assert(i == NumMemOps-1 && i != 0); 5638 DstOff -= VTSize - Size; 5639 } 5640 5641 // If this store is smaller than the largest store see whether we can get 5642 // the smaller value for free with a truncate. 5643 SDValue Value = MemSetValue; 5644 if (VT.bitsLT(LargestVT)) { 5645 if (!LargestVT.isVector() && !VT.isVector() && 5646 TLI.isTruncateFree(LargestVT, VT)) 5647 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); 5648 else 5649 Value = getMemsetValue(Src, VT, DAG, dl); 5650 } 5651 assert(Value.getValueType() == VT && "Value with wrong type."); 5652 SDValue Store = DAG.getStore( 5653 Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl), 5654 DstPtrInfo.getWithOffset(DstOff), Align, 5655 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone); 5656 OutChains.push_back(Store); 5657 DstOff += VT.getSizeInBits() / 8; 5658 Size -= VTSize; 5659 } 5660 5661 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); 5662 } 5663 5664 static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, 5665 unsigned AS) { 5666 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all 5667 // pointer operands can be losslessly bitcasted to pointers of address space 0 5668 if (AS != 0 && !TLI->isNoopAddrSpaceCast(AS, 0)) { 5669 report_fatal_error("cannot lower memory intrinsic in address space " + 5670 Twine(AS)); 5671 } 5672 } 5673 5674 SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, 5675 SDValue Src, SDValue Size, unsigned Align, 5676 bool isVol, bool AlwaysInline, bool isTailCall, 5677 MachinePointerInfo DstPtrInfo, 5678 MachinePointerInfo SrcPtrInfo) { 5679 assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 5680 5681 // Check to see if we should lower the memcpy to loads and stores first. 5682 // For cases within the target-specified limits, this is the best choice. 5683 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 5684 if (ConstantSize) { 5685 // Memcpy with size zero? Just return the original chain. 5686 if (ConstantSize->isNullValue()) 5687 return Chain; 5688 5689 SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 5690 ConstantSize->getZExtValue(),Align, 5691 isVol, false, DstPtrInfo, SrcPtrInfo); 5692 if (Result.getNode()) 5693 return Result; 5694 } 5695 5696 // Then check to see if we should lower the memcpy with target-specific 5697 // code. If the target chooses to do this, this is the next best. 5698 if (TSI) { 5699 SDValue Result = TSI->EmitTargetCodeForMemcpy( 5700 *this, dl, Chain, Dst, Src, Size, Align, isVol, AlwaysInline, 5701 DstPtrInfo, SrcPtrInfo); 5702 if (Result.getNode()) 5703 return Result; 5704 } 5705 5706 // If we really need inline code and the target declined to provide it, 5707 // use a (potentially long) sequence of loads and stores. 5708 if (AlwaysInline) { 5709 assert(ConstantSize && "AlwaysInline requires a constant size!"); 5710 return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 5711 ConstantSize->getZExtValue(), Align, isVol, 5712 true, DstPtrInfo, SrcPtrInfo); 5713 } 5714 5715 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); 5716 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace()); 5717 5718 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc 5719 // memcpy is not guaranteed to be safe. libc memcpys aren't required to 5720 // respect volatile, so they may do things like read or write memory 5721 // beyond the given memory regions. But fixing this isn't easy, and most 5722 // people don't care. 5723 5724 // Emit a library call. 5725 TargetLowering::ArgListTy Args; 5726 TargetLowering::ArgListEntry Entry; 5727 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 5728 Entry.Node = Dst; Args.push_back(Entry); 5729 Entry.Node = Src; Args.push_back(Entry); 5730 Entry.Node = Size; Args.push_back(Entry); 5731 // FIXME: pass in SDLoc 5732 TargetLowering::CallLoweringInfo CLI(*this); 5733 CLI.setDebugLoc(dl) 5734 .setChain(Chain) 5735 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY), 5736 Dst.getValueType().getTypeForEVT(*getContext()), 5737 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY), 5738 TLI->getPointerTy(getDataLayout())), 5739 std::move(Args)) 5740 .setDiscardResult() 5741 .setTailCall(isTailCall); 5742 5743 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 5744 return CallResult.second; 5745 } 5746 5747 SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl, 5748 SDValue Dst, unsigned DstAlign, 5749 SDValue Src, unsigned SrcAlign, 5750 SDValue Size, Type *SizeTy, 5751 unsigned ElemSz, bool isTailCall, 5752 MachinePointerInfo DstPtrInfo, 5753 MachinePointerInfo SrcPtrInfo) { 5754 // Emit a library call. 5755 TargetLowering::ArgListTy Args; 5756 TargetLowering::ArgListEntry Entry; 5757 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 5758 Entry.Node = Dst; 5759 Args.push_back(Entry); 5760 5761 Entry.Node = Src; 5762 Args.push_back(Entry); 5763 5764 Entry.Ty = SizeTy; 5765 Entry.Node = Size; 5766 Args.push_back(Entry); 5767 5768 RTLIB::Libcall LibraryCall = 5769 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElemSz); 5770 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) 5771 report_fatal_error("Unsupported element size"); 5772 5773 TargetLowering::CallLoweringInfo CLI(*this); 5774 CLI.setDebugLoc(dl) 5775 .setChain(Chain) 5776 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall), 5777 Type::getVoidTy(*getContext()), 5778 getExternalSymbol(TLI->getLibcallName(LibraryCall), 5779 TLI->getPointerTy(getDataLayout())), 5780 std::move(Args)) 5781 .setDiscardResult() 5782 .setTailCall(isTailCall); 5783 5784 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI); 5785 return CallResult.second; 5786 } 5787 5788 SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, 5789 SDValue Src, SDValue Size, unsigned Align, 5790 bool isVol, bool isTailCall, 5791 MachinePointerInfo DstPtrInfo, 5792 MachinePointerInfo SrcPtrInfo) { 5793 assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 5794 5795 // Check to see if we should lower the memmove to loads and stores first. 5796 // For cases within the target-specified limits, this is the best choice. 5797 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 5798 if (ConstantSize) { 5799 // Memmove with size zero? Just return the original chain. 5800 if (ConstantSize->isNullValue()) 5801 return Chain; 5802 5803 SDValue Result = 5804 getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src, 5805 ConstantSize->getZExtValue(), Align, isVol, 5806 false, DstPtrInfo, SrcPtrInfo); 5807 if (Result.getNode()) 5808 return Result; 5809 } 5810 5811 // Then check to see if we should lower the memmove with target-specific 5812 // code. If the target chooses to do this, this is the next best. 5813 if (TSI) { 5814 SDValue Result = TSI->EmitTargetCodeForMemmove( 5815 *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo); 5816 if (Result.getNode()) 5817 return Result; 5818 } 5819 5820 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); 5821 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace()); 5822 5823 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may 5824 // not be safe. See memcpy above for more details. 5825 5826 // Emit a library call. 5827 TargetLowering::ArgListTy Args; 5828 TargetLowering::ArgListEntry Entry; 5829 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 5830 Entry.Node = Dst; Args.push_back(Entry); 5831 Entry.Node = Src; Args.push_back(Entry); 5832 Entry.Node = Size; Args.push_back(Entry); 5833 // FIXME: pass in SDLoc 5834 TargetLowering::CallLoweringInfo CLI(*this); 5835 CLI.setDebugLoc(dl) 5836 .setChain(Chain) 5837 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE), 5838 Dst.getValueType().getTypeForEVT(*getContext()), 5839 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE), 5840 TLI->getPointerTy(getDataLayout())), 5841 std::move(Args)) 5842 .setDiscardResult() 5843 .setTailCall(isTailCall); 5844 5845 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 5846 return CallResult.second; 5847 } 5848 5849 SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl, 5850 SDValue Dst, unsigned DstAlign, 5851 SDValue Src, unsigned SrcAlign, 5852 SDValue Size, Type *SizeTy, 5853 unsigned ElemSz, bool isTailCall, 5854 MachinePointerInfo DstPtrInfo, 5855 MachinePointerInfo SrcPtrInfo) { 5856 // Emit a library call. 5857 TargetLowering::ArgListTy Args; 5858 TargetLowering::ArgListEntry Entry; 5859 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 5860 Entry.Node = Dst; 5861 Args.push_back(Entry); 5862 5863 Entry.Node = Src; 5864 Args.push_back(Entry); 5865 5866 Entry.Ty = SizeTy; 5867 Entry.Node = Size; 5868 Args.push_back(Entry); 5869 5870 RTLIB::Libcall LibraryCall = 5871 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElemSz); 5872 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) 5873 report_fatal_error("Unsupported element size"); 5874 5875 TargetLowering::CallLoweringInfo CLI(*this); 5876 CLI.setDebugLoc(dl) 5877 .setChain(Chain) 5878 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall), 5879 Type::getVoidTy(*getContext()), 5880 getExternalSymbol(TLI->getLibcallName(LibraryCall), 5881 TLI->getPointerTy(getDataLayout())), 5882 std::move(Args)) 5883 .setDiscardResult() 5884 .setTailCall(isTailCall); 5885 5886 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI); 5887 return CallResult.second; 5888 } 5889 5890 SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, 5891 SDValue Src, SDValue Size, unsigned Align, 5892 bool isVol, bool isTailCall, 5893 MachinePointerInfo DstPtrInfo) { 5894 assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 5895 5896 // Check to see if we should lower the memset to stores first. 5897 // For cases within the target-specified limits, this is the best choice. 5898 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 5899 if (ConstantSize) { 5900 // Memset with size zero? Just return the original chain. 5901 if (ConstantSize->isNullValue()) 5902 return Chain; 5903 5904 SDValue Result = 5905 getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), 5906 Align, isVol, DstPtrInfo); 5907 5908 if (Result.getNode()) 5909 return Result; 5910 } 5911 5912 // Then check to see if we should lower the memset with target-specific 5913 // code. If the target chooses to do this, this is the next best. 5914 if (TSI) { 5915 SDValue Result = TSI->EmitTargetCodeForMemset( 5916 *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo); 5917 if (Result.getNode()) 5918 return Result; 5919 } 5920 5921 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); 5922 5923 // Emit a library call. 5924 Type *IntPtrTy = getDataLayout().getIntPtrType(*getContext()); 5925 TargetLowering::ArgListTy Args; 5926 TargetLowering::ArgListEntry Entry; 5927 Entry.Node = Dst; Entry.Ty = IntPtrTy; 5928 Args.push_back(Entry); 5929 Entry.Node = Src; 5930 Entry.Ty = Src.getValueType().getTypeForEVT(*getContext()); 5931 Args.push_back(Entry); 5932 Entry.Node = Size; 5933 Entry.Ty = IntPtrTy; 5934 Args.push_back(Entry); 5935 5936 // FIXME: pass in SDLoc 5937 TargetLowering::CallLoweringInfo CLI(*this); 5938 CLI.setDebugLoc(dl) 5939 .setChain(Chain) 5940 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET), 5941 Dst.getValueType().getTypeForEVT(*getContext()), 5942 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET), 5943 TLI->getPointerTy(getDataLayout())), 5944 std::move(Args)) 5945 .setDiscardResult() 5946 .setTailCall(isTailCall); 5947 5948 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 5949 return CallResult.second; 5950 } 5951 5952 SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl, 5953 SDValue Dst, unsigned DstAlign, 5954 SDValue Value, SDValue Size, Type *SizeTy, 5955 unsigned ElemSz, bool isTailCall, 5956 MachinePointerInfo DstPtrInfo) { 5957 // Emit a library call. 5958 TargetLowering::ArgListTy Args; 5959 TargetLowering::ArgListEntry Entry; 5960 Entry.Ty = getDataLayout().getIntPtrType(*getContext()); 5961 Entry.Node = Dst; 5962 Args.push_back(Entry); 5963 5964 Entry.Ty = Type::getInt8Ty(*getContext()); 5965 Entry.Node = Value; 5966 Args.push_back(Entry); 5967 5968 Entry.Ty = SizeTy; 5969 Entry.Node = Size; 5970 Args.push_back(Entry); 5971 5972 RTLIB::Libcall LibraryCall = 5973 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElemSz); 5974 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) 5975 report_fatal_error("Unsupported element size"); 5976 5977 TargetLowering::CallLoweringInfo CLI(*this); 5978 CLI.setDebugLoc(dl) 5979 .setChain(Chain) 5980 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall), 5981 Type::getVoidTy(*getContext()), 5982 getExternalSymbol(TLI->getLibcallName(LibraryCall), 5983 TLI->getPointerTy(getDataLayout())), 5984 std::move(Args)) 5985 .setDiscardResult() 5986 .setTailCall(isTailCall); 5987 5988 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI); 5989 return CallResult.second; 5990 } 5991 5992 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, 5993 SDVTList VTList, ArrayRef<SDValue> Ops, 5994 MachineMemOperand *MMO) { 5995 FoldingSetNodeID ID; 5996 ID.AddInteger(MemVT.getRawBits()); 5997 AddNodeIDNode(ID, Opcode, VTList, Ops); 5998 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 5999 void* IP = nullptr; 6000 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 6001 cast<AtomicSDNode>(E)->refineAlignment(MMO); 6002 return SDValue(E, 0); 6003 } 6004 6005 auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), 6006 VTList, MemVT, MMO); 6007 createOperands(N, Ops); 6008 6009 CSEMap.InsertNode(N, IP); 6010 InsertNode(N); 6011 return SDValue(N, 0); 6012 } 6013 6014 SDValue SelectionDAG::getAtomicCmpSwap( 6015 unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, 6016 SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, 6017 unsigned Alignment, AtomicOrdering SuccessOrdering, 6018 AtomicOrdering FailureOrdering, SyncScope::ID SSID) { 6019 assert(Opcode == ISD::ATOMIC_CMP_SWAP || 6020 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); 6021 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); 6022 6023 if (Alignment == 0) // Ensure that codegen never sees alignment 0 6024 Alignment = getEVTAlignment(MemVT); 6025 6026 MachineFunction &MF = getMachineFunction(); 6027 6028 // FIXME: Volatile isn't really correct; we should keep track of atomic 6029 // orderings in the memoperand. 6030 auto Flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad | 6031 MachineMemOperand::MOStore; 6032 MachineMemOperand *MMO = 6033 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, 6034 AAMDNodes(), nullptr, SSID, SuccessOrdering, 6035 FailureOrdering); 6036 6037 return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO); 6038 } 6039 6040 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, 6041 EVT MemVT, SDVTList VTs, SDValue Chain, 6042 SDValue Ptr, SDValue Cmp, SDValue Swp, 6043 MachineMemOperand *MMO) { 6044 assert(Opcode == ISD::ATOMIC_CMP_SWAP || 6045 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); 6046 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); 6047 6048 SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; 6049 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); 6050 } 6051 6052 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, 6053 SDValue Chain, SDValue Ptr, SDValue Val, 6054 const Value *PtrVal, unsigned Alignment, 6055 AtomicOrdering Ordering, 6056 SyncScope::ID SSID) { 6057 if (Alignment == 0) // Ensure that codegen never sees alignment 0 6058 Alignment = getEVTAlignment(MemVT); 6059 6060 MachineFunction &MF = getMachineFunction(); 6061 // An atomic store does not load. An atomic load does not store. 6062 // (An atomicrmw obviously both loads and stores.) 6063 // For now, atomics are considered to be volatile always, and they are 6064 // chained as such. 6065 // FIXME: Volatile isn't really correct; we should keep track of atomic 6066 // orderings in the memoperand. 6067 auto Flags = MachineMemOperand::MOVolatile; 6068 if (Opcode != ISD::ATOMIC_STORE) 6069 Flags |= MachineMemOperand::MOLoad; 6070 if (Opcode != ISD::ATOMIC_LOAD) 6071 Flags |= MachineMemOperand::MOStore; 6072 6073 MachineMemOperand *MMO = 6074 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 6075 MemVT.getStoreSize(), Alignment, AAMDNodes(), 6076 nullptr, SSID, Ordering); 6077 6078 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO); 6079 } 6080 6081 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, 6082 SDValue Chain, SDValue Ptr, SDValue Val, 6083 MachineMemOperand *MMO) { 6084 assert((Opcode == ISD::ATOMIC_LOAD_ADD || 6085 Opcode == ISD::ATOMIC_LOAD_SUB || 6086 Opcode == ISD::ATOMIC_LOAD_AND || 6087 Opcode == ISD::ATOMIC_LOAD_CLR || 6088 Opcode == ISD::ATOMIC_LOAD_OR || 6089 Opcode == ISD::ATOMIC_LOAD_XOR || 6090 Opcode == ISD::ATOMIC_LOAD_NAND || 6091 Opcode == ISD::ATOMIC_LOAD_MIN || 6092 Opcode == ISD::ATOMIC_LOAD_MAX || 6093 Opcode == ISD::ATOMIC_LOAD_UMIN || 6094 Opcode == ISD::ATOMIC_LOAD_UMAX || 6095 Opcode == ISD::ATOMIC_SWAP || 6096 Opcode == ISD::ATOMIC_STORE) && 6097 "Invalid Atomic Op"); 6098 6099 EVT VT = Val.getValueType(); 6100 6101 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : 6102 getVTList(VT, MVT::Other); 6103 SDValue Ops[] = {Chain, Ptr, Val}; 6104 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); 6105 } 6106 6107 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, 6108 EVT VT, SDValue Chain, SDValue Ptr, 6109 MachineMemOperand *MMO) { 6110 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); 6111 6112 SDVTList VTs = getVTList(VT, MVT::Other); 6113 SDValue Ops[] = {Chain, Ptr}; 6114 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); 6115 } 6116 6117 /// getMergeValues - Create a MERGE_VALUES node from the given operands. 6118 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) { 6119 if (Ops.size() == 1) 6120 return Ops[0]; 6121 6122 SmallVector<EVT, 4> VTs; 6123 VTs.reserve(Ops.size()); 6124 for (unsigned i = 0; i < Ops.size(); ++i) 6125 VTs.push_back(Ops[i].getValueType()); 6126 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops); 6127 } 6128 6129 SDValue SelectionDAG::getMemIntrinsicNode( 6130 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops, 6131 EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align, 6132 MachineMemOperand::Flags Flags, unsigned Size) { 6133 if (Align == 0) // Ensure that codegen never sees alignment 0 6134 Align = getEVTAlignment(MemVT); 6135 6136 if (!Size) 6137 Size = MemVT.getStoreSize(); 6138 6139 MachineFunction &MF = getMachineFunction(); 6140 MachineMemOperand *MMO = 6141 MF.getMachineMemOperand(PtrInfo, Flags, Size, Align); 6142 6143 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO); 6144 } 6145 6146 SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, 6147 SDVTList VTList, 6148 ArrayRef<SDValue> Ops, EVT MemVT, 6149 MachineMemOperand *MMO) { 6150 assert((Opcode == ISD::INTRINSIC_VOID || 6151 Opcode == ISD::INTRINSIC_W_CHAIN || 6152 Opcode == ISD::PREFETCH || 6153 Opcode == ISD::LIFETIME_START || 6154 Opcode == ISD::LIFETIME_END || 6155 ((int)Opcode <= std::numeric_limits<int>::max() && 6156 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && 6157 "Opcode is not a memory-accessing opcode!"); 6158 6159 // Memoize the node unless it returns a flag. 6160 MemIntrinsicSDNode *N; 6161 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 6162 FoldingSetNodeID ID; 6163 AddNodeIDNode(ID, Opcode, VTList, Ops); 6164 ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>( 6165 Opcode, dl.getIROrder(), VTList, MemVT, MMO)); 6166 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 6167 void *IP = nullptr; 6168 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 6169 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO); 6170 return SDValue(E, 0); 6171 } 6172 6173 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), 6174 VTList, MemVT, MMO); 6175 createOperands(N, Ops); 6176 6177 CSEMap.InsertNode(N, IP); 6178 } else { 6179 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), 6180 VTList, MemVT, MMO); 6181 createOperands(N, Ops); 6182 } 6183 InsertNode(N); 6184 return SDValue(N, 0); 6185 } 6186 6187 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 6188 /// MachinePointerInfo record from it. This is particularly useful because the 6189 /// code generator has many cases where it doesn't bother passing in a 6190 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 6191 static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, 6192 SelectionDAG &DAG, SDValue Ptr, 6193 int64_t Offset = 0) { 6194 // If this is FI+Offset, we can model it. 6195 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) 6196 return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), 6197 FI->getIndex(), Offset); 6198 6199 // If this is (FI+Offset1)+Offset2, we can model it. 6200 if (Ptr.getOpcode() != ISD::ADD || 6201 !isa<ConstantSDNode>(Ptr.getOperand(1)) || 6202 !isa<FrameIndexSDNode>(Ptr.getOperand(0))) 6203 return Info; 6204 6205 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 6206 return MachinePointerInfo::getFixedStack( 6207 DAG.getMachineFunction(), FI, 6208 Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue()); 6209 } 6210 6211 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 6212 /// MachinePointerInfo record from it. This is particularly useful because the 6213 /// code generator has many cases where it doesn't bother passing in a 6214 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 6215 static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, 6216 SelectionDAG &DAG, SDValue Ptr, 6217 SDValue OffsetOp) { 6218 // If the 'Offset' value isn't a constant, we can't handle this. 6219 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp)) 6220 return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue()); 6221 if (OffsetOp.isUndef()) 6222 return InferPointerInfo(Info, DAG, Ptr); 6223 return Info; 6224 } 6225 6226 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 6227 EVT VT, const SDLoc &dl, SDValue Chain, 6228 SDValue Ptr, SDValue Offset, 6229 MachinePointerInfo PtrInfo, EVT MemVT, 6230 unsigned Alignment, 6231 MachineMemOperand::Flags MMOFlags, 6232 const AAMDNodes &AAInfo, const MDNode *Ranges) { 6233 assert(Chain.getValueType() == MVT::Other && 6234 "Invalid chain type"); 6235 if (Alignment == 0) // Ensure that codegen never sees alignment 0 6236 Alignment = getEVTAlignment(MemVT); 6237 6238 MMOFlags |= MachineMemOperand::MOLoad; 6239 assert((MMOFlags & MachineMemOperand::MOStore) == 0); 6240 // If we don't have a PtrInfo, infer the trivial frame index case to simplify 6241 // clients. 6242 if (PtrInfo.V.isNull()) 6243 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset); 6244 6245 MachineFunction &MF = getMachineFunction(); 6246 MachineMemOperand *MMO = MF.getMachineMemOperand( 6247 PtrInfo, MMOFlags, MemVT.getStoreSize(), Alignment, AAInfo, Ranges); 6248 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); 6249 } 6250 6251 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 6252 EVT VT, const SDLoc &dl, SDValue Chain, 6253 SDValue Ptr, SDValue Offset, EVT MemVT, 6254 MachineMemOperand *MMO) { 6255 if (VT == MemVT) { 6256 ExtType = ISD::NON_EXTLOAD; 6257 } else if (ExtType == ISD::NON_EXTLOAD) { 6258 assert(VT == MemVT && "Non-extending load from different memory type!"); 6259 } else { 6260 // Extending load. 6261 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && 6262 "Should only be an extending load, not truncating!"); 6263 assert(VT.isInteger() == MemVT.isInteger() && 6264 "Cannot convert from FP to Int or Int -> FP!"); 6265 assert(VT.isVector() == MemVT.isVector() && 6266 "Cannot use an ext load to convert to or from a vector!"); 6267 assert((!VT.isVector() || 6268 VT.getVectorNumElements() == MemVT.getVectorNumElements()) && 6269 "Cannot use an ext load to change the number of vector elements!"); 6270 } 6271 6272 bool Indexed = AM != ISD::UNINDEXED; 6273 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!"); 6274 6275 SDVTList VTs = Indexed ? 6276 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); 6277 SDValue Ops[] = { Chain, Ptr, Offset }; 6278 FoldingSetNodeID ID; 6279 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops); 6280 ID.AddInteger(MemVT.getRawBits()); 6281 ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>( 6282 dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO)); 6283 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 6284 void *IP = nullptr; 6285 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 6286 cast<LoadSDNode>(E)->refineAlignment(MMO); 6287 return SDValue(E, 0); 6288 } 6289 auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, 6290 ExtType, MemVT, MMO); 6291 createOperands(N, Ops); 6292 6293 CSEMap.InsertNode(N, IP); 6294 InsertNode(N); 6295 SDValue V(N, 0); 6296 NewSDValueDbgMsg(V, "Creating new node: ", this); 6297 return V; 6298 } 6299 6300 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain, 6301 SDValue Ptr, MachinePointerInfo PtrInfo, 6302 unsigned Alignment, 6303 MachineMemOperand::Flags MMOFlags, 6304 const AAMDNodes &AAInfo, const MDNode *Ranges) { 6305 SDValue Undef = getUNDEF(Ptr.getValueType()); 6306 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 6307 PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges); 6308 } 6309 6310 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain, 6311 SDValue Ptr, MachineMemOperand *MMO) { 6312 SDValue Undef = getUNDEF(Ptr.getValueType()); 6313 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 6314 VT, MMO); 6315 } 6316 6317 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, 6318 EVT VT, SDValue Chain, SDValue Ptr, 6319 MachinePointerInfo PtrInfo, EVT MemVT, 6320 unsigned Alignment, 6321 MachineMemOperand::Flags MMOFlags, 6322 const AAMDNodes &AAInfo) { 6323 SDValue Undef = getUNDEF(Ptr.getValueType()); 6324 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo, 6325 MemVT, Alignment, MMOFlags, AAInfo); 6326 } 6327 6328 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, 6329 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT, 6330 MachineMemOperand *MMO) { 6331 SDValue Undef = getUNDEF(Ptr.getValueType()); 6332 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 6333 MemVT, MMO); 6334 } 6335 6336 SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, 6337 SDValue Base, SDValue Offset, 6338 ISD::MemIndexedMode AM) { 6339 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad); 6340 assert(LD->getOffset().isUndef() && "Load is already a indexed load!"); 6341 // Don't propagate the invariant or dereferenceable flags. 6342 auto MMOFlags = 6343 LD->getMemOperand()->getFlags() & 6344 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); 6345 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, 6346 LD->getChain(), Base, Offset, LD->getPointerInfo(), 6347 LD->getMemoryVT(), LD->getAlignment(), MMOFlags, 6348 LD->getAAInfo()); 6349 } 6350 6351 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, 6352 SDValue Ptr, MachinePointerInfo PtrInfo, 6353 unsigned Alignment, 6354 MachineMemOperand::Flags MMOFlags, 6355 const AAMDNodes &AAInfo) { 6356 assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); 6357 if (Alignment == 0) // Ensure that codegen never sees alignment 0 6358 Alignment = getEVTAlignment(Val.getValueType()); 6359 6360 MMOFlags |= MachineMemOperand::MOStore; 6361 assert((MMOFlags & MachineMemOperand::MOLoad) == 0); 6362 6363 if (PtrInfo.V.isNull()) 6364 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); 6365 6366 MachineFunction &MF = getMachineFunction(); 6367 MachineMemOperand *MMO = MF.getMachineMemOperand( 6368 PtrInfo, MMOFlags, Val.getValueType().getStoreSize(), Alignment, AAInfo); 6369 return getStore(Chain, dl, Val, Ptr, MMO); 6370 } 6371 6372 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, 6373 SDValue Ptr, MachineMemOperand *MMO) { 6374 assert(Chain.getValueType() == MVT::Other && 6375 "Invalid chain type"); 6376 EVT VT = Val.getValueType(); 6377 SDVTList VTs = getVTList(MVT::Other); 6378 SDValue Undef = getUNDEF(Ptr.getValueType()); 6379 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 6380 FoldingSetNodeID ID; 6381 AddNodeIDNode(ID, ISD::STORE, VTs, Ops); 6382 ID.AddInteger(VT.getRawBits()); 6383 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>( 6384 dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO)); 6385 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 6386 void *IP = nullptr; 6387 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 6388 cast<StoreSDNode>(E)->refineAlignment(MMO); 6389 return SDValue(E, 0); 6390 } 6391 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, 6392 ISD::UNINDEXED, false, VT, MMO); 6393 createOperands(N, Ops); 6394 6395 CSEMap.InsertNode(N, IP); 6396 InsertNode(N); 6397 SDValue V(N, 0); 6398 NewSDValueDbgMsg(V, "Creating new node: ", this); 6399 return V; 6400 } 6401 6402 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, 6403 SDValue Ptr, MachinePointerInfo PtrInfo, 6404 EVT SVT, unsigned Alignment, 6405 MachineMemOperand::Flags MMOFlags, 6406 const AAMDNodes &AAInfo) { 6407 assert(Chain.getValueType() == MVT::Other && 6408 "Invalid chain type"); 6409 if (Alignment == 0) // Ensure that codegen never sees alignment 0 6410 Alignment = getEVTAlignment(SVT); 6411 6412 MMOFlags |= MachineMemOperand::MOStore; 6413 assert((MMOFlags & MachineMemOperand::MOLoad) == 0); 6414 6415 if (PtrInfo.V.isNull()) 6416 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); 6417 6418 MachineFunction &MF = getMachineFunction(); 6419 MachineMemOperand *MMO = MF.getMachineMemOperand( 6420 PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo); 6421 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); 6422 } 6423 6424 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, 6425 SDValue Ptr, EVT SVT, 6426 MachineMemOperand *MMO) { 6427 EVT VT = Val.getValueType(); 6428 6429 assert(Chain.getValueType() == MVT::Other && 6430 "Invalid chain type"); 6431 if (VT == SVT) 6432 return getStore(Chain, dl, Val, Ptr, MMO); 6433 6434 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && 6435 "Should only be a truncating store, not extending!"); 6436 assert(VT.isInteger() == SVT.isInteger() && 6437 "Can't do FP-INT conversion!"); 6438 assert(VT.isVector() == SVT.isVector() && 6439 "Cannot use trunc store to convert to or from a vector!"); 6440 assert((!VT.isVector() || 6441 VT.getVectorNumElements() == SVT.getVectorNumElements()) && 6442 "Cannot use trunc store to change the number of vector elements!"); 6443 6444 SDVTList VTs = getVTList(MVT::Other); 6445 SDValue Undef = getUNDEF(Ptr.getValueType()); 6446 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 6447 FoldingSetNodeID ID; 6448 AddNodeIDNode(ID, ISD::STORE, VTs, Ops); 6449 ID.AddInteger(SVT.getRawBits()); 6450 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>( 6451 dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO)); 6452 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 6453 void *IP = nullptr; 6454 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 6455 cast<StoreSDNode>(E)->refineAlignment(MMO); 6456 return SDValue(E, 0); 6457 } 6458 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, 6459 ISD::UNINDEXED, true, SVT, MMO); 6460 createOperands(N, Ops); 6461 6462 CSEMap.InsertNode(N, IP); 6463 InsertNode(N); 6464 SDValue V(N, 0); 6465 NewSDValueDbgMsg(V, "Creating new node: ", this); 6466 return V; 6467 } 6468 6469 SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl, 6470 SDValue Base, SDValue Offset, 6471 ISD::MemIndexedMode AM) { 6472 StoreSDNode *ST = cast<StoreSDNode>(OrigStore); 6473 assert(ST->getOffset().isUndef() && "Store is already a indexed store!"); 6474 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); 6475 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; 6476 FoldingSetNodeID ID; 6477 AddNodeIDNode(ID, ISD::STORE, VTs, Ops); 6478 ID.AddInteger(ST->getMemoryVT().getRawBits()); 6479 ID.AddInteger(ST->getRawSubclassData()); 6480 ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 6481 void *IP = nullptr; 6482 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) 6483 return SDValue(E, 0); 6484 6485 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, 6486 ST->isTruncatingStore(), ST->getMemoryVT(), 6487 ST->getMemOperand()); 6488 createOperands(N, Ops); 6489 6490 CSEMap.InsertNode(N, IP); 6491 InsertNode(N); 6492 SDValue V(N, 0); 6493 NewSDValueDbgMsg(V, "Creating new node: ", this); 6494 return V; 6495 } 6496 6497 SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, 6498 SDValue Ptr, SDValue Mask, SDValue Src0, 6499 EVT MemVT, MachineMemOperand *MMO, 6500 ISD::LoadExtType ExtTy, bool isExpanding) { 6501 SDVTList VTs = getVTList(VT, MVT::Other); 6502 SDValue Ops[] = { Chain, Ptr, Mask, Src0 }; 6503 FoldingSetNodeID ID; 6504 AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops); 6505 ID.AddInteger(VT.getRawBits()); 6506 ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>( 6507 dl.getIROrder(), VTs, ExtTy, isExpanding, MemVT, MMO)); 6508 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 6509 void *IP = nullptr; 6510 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 6511 cast<MaskedLoadSDNode>(E)->refineAlignment(MMO); 6512 return SDValue(E, 0); 6513 } 6514 auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, 6515 ExtTy, isExpanding, MemVT, MMO); 6516 createOperands(N, Ops); 6517 6518 CSEMap.InsertNode(N, IP); 6519 InsertNode(N); 6520 SDValue V(N, 0); 6521 NewSDValueDbgMsg(V, "Creating new node: ", this); 6522 return V; 6523 } 6524 6525 SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl, 6526 SDValue Val, SDValue Ptr, SDValue Mask, 6527 EVT MemVT, MachineMemOperand *MMO, 6528 bool IsTruncating, bool IsCompressing) { 6529 assert(Chain.getValueType() == MVT::Other && 6530 "Invalid chain type"); 6531 EVT VT = Val.getValueType(); 6532 SDVTList VTs = getVTList(MVT::Other); 6533 SDValue Ops[] = { Chain, Ptr, Mask, Val }; 6534 FoldingSetNodeID ID; 6535 AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops); 6536 ID.AddInteger(VT.getRawBits()); 6537 ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>( 6538 dl.getIROrder(), VTs, IsTruncating, IsCompressing, MemVT, MMO)); 6539 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 6540 void *IP = nullptr; 6541 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 6542 cast<MaskedStoreSDNode>(E)->refineAlignment(MMO); 6543 return SDValue(E, 0); 6544 } 6545 auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, 6546 IsTruncating, IsCompressing, MemVT, MMO); 6547 createOperands(N, Ops); 6548 6549 CSEMap.InsertNode(N, IP); 6550 InsertNode(N); 6551 SDValue V(N, 0); 6552 NewSDValueDbgMsg(V, "Creating new node: ", this); 6553 return V; 6554 } 6555 6556 SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, 6557 ArrayRef<SDValue> Ops, 6558 MachineMemOperand *MMO) { 6559 assert(Ops.size() == 6 && "Incompatible number of operands"); 6560 6561 FoldingSetNodeID ID; 6562 AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops); 6563 ID.AddInteger(VT.getRawBits()); 6564 ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>( 6565 dl.getIROrder(), VTs, VT, MMO)); 6566 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 6567 void *IP = nullptr; 6568 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 6569 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO); 6570 return SDValue(E, 0); 6571 } 6572 6573 auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(), 6574 VTs, VT, MMO); 6575 createOperands(N, Ops); 6576 6577 assert(N->getValue().getValueType() == N->getValueType(0) && 6578 "Incompatible type of the PassThru value in MaskedGatherSDNode"); 6579 assert(N->getMask().getValueType().getVectorNumElements() == 6580 N->getValueType(0).getVectorNumElements() && 6581 "Vector width mismatch between mask and data"); 6582 assert(N->getIndex().getValueType().getVectorNumElements() == 6583 N->getValueType(0).getVectorNumElements() && 6584 "Vector width mismatch between index and data"); 6585 assert(isa<ConstantSDNode>(N->getScale()) && 6586 cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() && 6587 "Scale should be a constant power of 2"); 6588 6589 CSEMap.InsertNode(N, IP); 6590 InsertNode(N); 6591 SDValue V(N, 0); 6592 NewSDValueDbgMsg(V, "Creating new node: ", this); 6593 return V; 6594 } 6595 6596 SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, 6597 ArrayRef<SDValue> Ops, 6598 MachineMemOperand *MMO) { 6599 assert(Ops.size() == 6 && "Incompatible number of operands"); 6600 6601 FoldingSetNodeID ID; 6602 AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); 6603 ID.AddInteger(VT.getRawBits()); 6604 ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>( 6605 dl.getIROrder(), VTs, VT, MMO)); 6606 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 6607 void *IP = nullptr; 6608 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { 6609 cast<MaskedScatterSDNode>(E)->refineAlignment(MMO); 6610 return SDValue(E, 0); 6611 } 6612 auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(), 6613 VTs, VT, MMO); 6614 createOperands(N, Ops); 6615 6616 assert(N->getMask().getValueType().getVectorNumElements() == 6617 N->getValue().getValueType().getVectorNumElements() && 6618 "Vector width mismatch between mask and data"); 6619 assert(N->getIndex().getValueType().getVectorNumElements() == 6620 N->getValue().getValueType().getVectorNumElements() && 6621 "Vector width mismatch between index and data"); 6622 assert(isa<ConstantSDNode>(N->getScale()) && 6623 cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() && 6624 "Scale should be a constant power of 2"); 6625 6626 CSEMap.InsertNode(N, IP); 6627 InsertNode(N); 6628 SDValue V(N, 0); 6629 NewSDValueDbgMsg(V, "Creating new node: ", this); 6630 return V; 6631 } 6632 6633 SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, 6634 SDValue Ptr, SDValue SV, unsigned Align) { 6635 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) }; 6636 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops); 6637 } 6638 6639 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 6640 ArrayRef<SDUse> Ops) { 6641 switch (Ops.size()) { 6642 case 0: return getNode(Opcode, DL, VT); 6643 case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0])); 6644 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 6645 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 6646 default: break; 6647 } 6648 6649 // Copy from an SDUse array into an SDValue array for use with 6650 // the regular getNode logic. 6651 SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end()); 6652 return getNode(Opcode, DL, VT, NewOps); 6653 } 6654 6655 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, 6656 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) { 6657 unsigned NumOps = Ops.size(); 6658 switch (NumOps) { 6659 case 0: return getNode(Opcode, DL, VT); 6660 case 1: return getNode(Opcode, DL, VT, Ops[0], Flags); 6661 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags); 6662 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 6663 default: break; 6664 } 6665 6666 switch (Opcode) { 6667 default: break; 6668 case ISD::CONCAT_VECTORS: 6669 // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF. 6670 if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this)) 6671 return V; 6672 break; 6673 case ISD::SELECT_CC: 6674 assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); 6675 assert(Ops[0].getValueType() == Ops[1].getValueType() && 6676 "LHS and RHS of condition must have same type!"); 6677 assert(Ops[2].getValueType() == Ops[3].getValueType() && 6678 "True and False arms of SelectCC must have same type!"); 6679 assert(Ops[2].getValueType() == VT && 6680 "select_cc node must be of same type as true and false value!"); 6681 break; 6682 case ISD::BR_CC: 6683 assert(NumOps == 5 && "BR_CC takes 5 operands!"); 6684 assert(Ops[2].getValueType() == Ops[3].getValueType() && 6685 "LHS/RHS of comparison should match types!"); 6686 break; 6687 } 6688 6689 // Memoize nodes. 6690 SDNode *N; 6691 SDVTList VTs = getVTList(VT); 6692 6693 if (VT != MVT::Glue) { 6694 FoldingSetNodeID ID; 6695 AddNodeIDNode(ID, Opcode, VTs, Ops); 6696 void *IP = nullptr; 6697 6698 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) 6699 return SDValue(E, 0); 6700 6701 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 6702 createOperands(N, Ops); 6703 6704 CSEMap.InsertNode(N, IP); 6705 } else { 6706 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 6707 createOperands(N, Ops); 6708 } 6709 6710 InsertNode(N); 6711 SDValue V(N, 0); 6712 NewSDValueDbgMsg(V, "Creating new node: ", this); 6713 return V; 6714 } 6715 6716 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, 6717 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) { 6718 return getNode(Opcode, DL, getVTList(ResultTys), Ops); 6719 } 6720 6721 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 6722 ArrayRef<SDValue> Ops) { 6723 if (VTList.NumVTs == 1) 6724 return getNode(Opcode, DL, VTList.VTs[0], Ops); 6725 6726 #if 0 6727 switch (Opcode) { 6728 // FIXME: figure out how to safely handle things like 6729 // int foo(int x) { return 1 << (x & 255); } 6730 // int bar() { return foo(256); } 6731 case ISD::SRA_PARTS: 6732 case ISD::SRL_PARTS: 6733 case ISD::SHL_PARTS: 6734 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && 6735 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) 6736 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 6737 else if (N3.getOpcode() == ISD::AND) 6738 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { 6739 // If the and is only masking out bits that cannot effect the shift, 6740 // eliminate the and. 6741 unsigned NumBits = VT.getScalarSizeInBits()*2; 6742 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) 6743 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 6744 } 6745 break; 6746 } 6747 #endif 6748 6749 // Memoize the node unless it returns a flag. 6750 SDNode *N; 6751 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 6752 FoldingSetNodeID ID; 6753 AddNodeIDNode(ID, Opcode, VTList, Ops); 6754 void *IP = nullptr; 6755 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) 6756 return SDValue(E, 0); 6757 6758 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList); 6759 createOperands(N, Ops); 6760 CSEMap.InsertNode(N, IP); 6761 } else { 6762 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList); 6763 createOperands(N, Ops); 6764 } 6765 InsertNode(N); 6766 SDValue V(N, 0); 6767 NewSDValueDbgMsg(V, "Creating new node: ", this); 6768 return V; 6769 } 6770 6771 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, 6772 SDVTList VTList) { 6773 return getNode(Opcode, DL, VTList, None); 6774 } 6775 6776 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 6777 SDValue N1) { 6778 SDValue Ops[] = { N1 }; 6779 return getNode(Opcode, DL, VTList, Ops); 6780 } 6781 6782 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 6783 SDValue N1, SDValue N2) { 6784 SDValue Ops[] = { N1, N2 }; 6785 return getNode(Opcode, DL, VTList, Ops); 6786 } 6787 6788 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 6789 SDValue N1, SDValue N2, SDValue N3) { 6790 SDValue Ops[] = { N1, N2, N3 }; 6791 return getNode(Opcode, DL, VTList, Ops); 6792 } 6793 6794 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 6795 SDValue N1, SDValue N2, SDValue N3, SDValue N4) { 6796 SDValue Ops[] = { N1, N2, N3, N4 }; 6797 return getNode(Opcode, DL, VTList, Ops); 6798 } 6799 6800 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, 6801 SDValue N1, SDValue N2, SDValue N3, SDValue N4, 6802 SDValue N5) { 6803 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 6804 return getNode(Opcode, DL, VTList, Ops); 6805 } 6806 6807 SDVTList SelectionDAG::getVTList(EVT VT) { 6808 return makeVTList(SDNode::getValueTypeList(VT), 1); 6809 } 6810 6811 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { 6812 FoldingSetNodeID ID; 6813 ID.AddInteger(2U); 6814 ID.AddInteger(VT1.getRawBits()); 6815 ID.AddInteger(VT2.getRawBits()); 6816 6817 void *IP = nullptr; 6818 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 6819 if (!Result) { 6820 EVT *Array = Allocator.Allocate<EVT>(2); 6821 Array[0] = VT1; 6822 Array[1] = VT2; 6823 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2); 6824 VTListMap.InsertNode(Result, IP); 6825 } 6826 return Result->getSDVTList(); 6827 } 6828 6829 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { 6830 FoldingSetNodeID ID; 6831 ID.AddInteger(3U); 6832 ID.AddInteger(VT1.getRawBits()); 6833 ID.AddInteger(VT2.getRawBits()); 6834 ID.AddInteger(VT3.getRawBits()); 6835 6836 void *IP = nullptr; 6837 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 6838 if (!Result) { 6839 EVT *Array = Allocator.Allocate<EVT>(3); 6840 Array[0] = VT1; 6841 Array[1] = VT2; 6842 Array[2] = VT3; 6843 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3); 6844 VTListMap.InsertNode(Result, IP); 6845 } 6846 return Result->getSDVTList(); 6847 } 6848 6849 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { 6850 FoldingSetNodeID ID; 6851 ID.AddInteger(4U); 6852 ID.AddInteger(VT1.getRawBits()); 6853 ID.AddInteger(VT2.getRawBits()); 6854 ID.AddInteger(VT3.getRawBits()); 6855 ID.AddInteger(VT4.getRawBits()); 6856 6857 void *IP = nullptr; 6858 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 6859 if (!Result) { 6860 EVT *Array = Allocator.Allocate<EVT>(4); 6861 Array[0] = VT1; 6862 Array[1] = VT2; 6863 Array[2] = VT3; 6864 Array[3] = VT4; 6865 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4); 6866 VTListMap.InsertNode(Result, IP); 6867 } 6868 return Result->getSDVTList(); 6869 } 6870 6871 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) { 6872 unsigned NumVTs = VTs.size(); 6873 FoldingSetNodeID ID; 6874 ID.AddInteger(NumVTs); 6875 for (unsigned index = 0; index < NumVTs; index++) { 6876 ID.AddInteger(VTs[index].getRawBits()); 6877 } 6878 6879 void *IP = nullptr; 6880 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 6881 if (!Result) { 6882 EVT *Array = Allocator.Allocate<EVT>(NumVTs); 6883 std::copy(VTs.begin(), VTs.end(), Array); 6884 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs); 6885 VTListMap.InsertNode(Result, IP); 6886 } 6887 return Result->getSDVTList(); 6888 } 6889 6890 6891 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the 6892 /// specified operands. If the resultant node already exists in the DAG, 6893 /// this does not modify the specified node, instead it returns the node that 6894 /// already exists. If the resultant node does not exist in the DAG, the 6895 /// input node is returned. As a degenerate case, if you specify the same 6896 /// input operands as the node already has, the input node is returned. 6897 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { 6898 assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); 6899 6900 // Check to see if there is no change. 6901 if (Op == N->getOperand(0)) return N; 6902 6903 // See if the modified node already exists. 6904 void *InsertPos = nullptr; 6905 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) 6906 return Existing; 6907 6908 // Nope it doesn't. Remove the node from its current place in the maps. 6909 if (InsertPos) 6910 if (!RemoveNodeFromCSEMaps(N)) 6911 InsertPos = nullptr; 6912 6913 // Now we update the operands. 6914 N->OperandList[0].set(Op); 6915 6916 updateDivergence(N); 6917 // If this gets put into a CSE map, add it. 6918 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 6919 return N; 6920 } 6921 6922 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { 6923 assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); 6924 6925 // Check to see if there is no change. 6926 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1)) 6927 return N; // No operands changed, just return the input node. 6928 6929 // See if the modified node already exists. 6930 void *InsertPos = nullptr; 6931 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) 6932 return Existing; 6933 6934 // Nope it doesn't. Remove the node from its current place in the maps. 6935 if (InsertPos) 6936 if (!RemoveNodeFromCSEMaps(N)) 6937 InsertPos = nullptr; 6938 6939 // Now we update the operands. 6940 if (N->OperandList[0] != Op1) 6941 N->OperandList[0].set(Op1); 6942 if (N->OperandList[1] != Op2) 6943 N->OperandList[1].set(Op2); 6944 6945 updateDivergence(N); 6946 // If this gets put into a CSE map, add it. 6947 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 6948 return N; 6949 } 6950 6951 SDNode *SelectionDAG:: 6952 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { 6953 SDValue Ops[] = { Op1, Op2, Op3 }; 6954 return UpdateNodeOperands(N, Ops); 6955 } 6956 6957 SDNode *SelectionDAG:: 6958 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 6959 SDValue Op3, SDValue Op4) { 6960 SDValue Ops[] = { Op1, Op2, Op3, Op4 }; 6961 return UpdateNodeOperands(N, Ops); 6962 } 6963 6964 SDNode *SelectionDAG:: 6965 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 6966 SDValue Op3, SDValue Op4, SDValue Op5) { 6967 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; 6968 return UpdateNodeOperands(N, Ops); 6969 } 6970 6971 SDNode *SelectionDAG:: 6972 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) { 6973 unsigned NumOps = Ops.size(); 6974 assert(N->getNumOperands() == NumOps && 6975 "Update with wrong number of operands"); 6976 6977 // If no operands changed just return the input node. 6978 if (std::equal(Ops.begin(), Ops.end(), N->op_begin())) 6979 return N; 6980 6981 // See if the modified node already exists. 6982 void *InsertPos = nullptr; 6983 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos)) 6984 return Existing; 6985 6986 // Nope it doesn't. Remove the node from its current place in the maps. 6987 if (InsertPos) 6988 if (!RemoveNodeFromCSEMaps(N)) 6989 InsertPos = nullptr; 6990 6991 // Now we update the operands. 6992 for (unsigned i = 0; i != NumOps; ++i) 6993 if (N->OperandList[i] != Ops[i]) 6994 N->OperandList[i].set(Ops[i]); 6995 6996 updateDivergence(N); 6997 // If this gets put into a CSE map, add it. 6998 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 6999 return N; 7000 } 7001 7002 /// DropOperands - Release the operands and set this node to have 7003 /// zero operands. 7004 void SDNode::DropOperands() { 7005 // Unlike the code in MorphNodeTo that does this, we don't need to 7006 // watch for dead nodes here. 7007 for (op_iterator I = op_begin(), E = op_end(); I != E; ) { 7008 SDUse &Use = *I++; 7009 Use.set(SDValue()); 7010 } 7011 } 7012 7013 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a 7014 /// machine opcode. 7015 /// 7016 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7017 EVT VT) { 7018 SDVTList VTs = getVTList(VT); 7019 return SelectNodeTo(N, MachineOpc, VTs, None); 7020 } 7021 7022 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7023 EVT VT, SDValue Op1) { 7024 SDVTList VTs = getVTList(VT); 7025 SDValue Ops[] = { Op1 }; 7026 return SelectNodeTo(N, MachineOpc, VTs, Ops); 7027 } 7028 7029 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7030 EVT VT, SDValue Op1, 7031 SDValue Op2) { 7032 SDVTList VTs = getVTList(VT); 7033 SDValue Ops[] = { Op1, Op2 }; 7034 return SelectNodeTo(N, MachineOpc, VTs, Ops); 7035 } 7036 7037 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7038 EVT VT, SDValue Op1, 7039 SDValue Op2, SDValue Op3) { 7040 SDVTList VTs = getVTList(VT); 7041 SDValue Ops[] = { Op1, Op2, Op3 }; 7042 return SelectNodeTo(N, MachineOpc, VTs, Ops); 7043 } 7044 7045 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7046 EVT VT, ArrayRef<SDValue> Ops) { 7047 SDVTList VTs = getVTList(VT); 7048 return SelectNodeTo(N, MachineOpc, VTs, Ops); 7049 } 7050 7051 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7052 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) { 7053 SDVTList VTs = getVTList(VT1, VT2); 7054 return SelectNodeTo(N, MachineOpc, VTs, Ops); 7055 } 7056 7057 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7058 EVT VT1, EVT VT2) { 7059 SDVTList VTs = getVTList(VT1, VT2); 7060 return SelectNodeTo(N, MachineOpc, VTs, None); 7061 } 7062 7063 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7064 EVT VT1, EVT VT2, EVT VT3, 7065 ArrayRef<SDValue> Ops) { 7066 SDVTList VTs = getVTList(VT1, VT2, VT3); 7067 return SelectNodeTo(N, MachineOpc, VTs, Ops); 7068 } 7069 7070 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7071 EVT VT1, EVT VT2, 7072 SDValue Op1, SDValue Op2) { 7073 SDVTList VTs = getVTList(VT1, VT2); 7074 SDValue Ops[] = { Op1, Op2 }; 7075 return SelectNodeTo(N, MachineOpc, VTs, Ops); 7076 } 7077 7078 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 7079 SDVTList VTs,ArrayRef<SDValue> Ops) { 7080 SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops); 7081 // Reset the NodeID to -1. 7082 New->setNodeId(-1); 7083 if (New != N) { 7084 ReplaceAllUsesWith(N, New); 7085 RemoveDeadNode(N); 7086 } 7087 return New; 7088 } 7089 7090 /// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away 7091 /// the line number information on the merged node since it is not possible to 7092 /// preserve the information that operation is associated with multiple lines. 7093 /// This will make the debugger working better at -O0, were there is a higher 7094 /// probability having other instructions associated with that line. 7095 /// 7096 /// For IROrder, we keep the smaller of the two 7097 SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) { 7098 DebugLoc NLoc = N->getDebugLoc(); 7099 if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) { 7100 N->setDebugLoc(DebugLoc()); 7101 } 7102 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder()); 7103 N->setIROrder(Order); 7104 return N; 7105 } 7106 7107 /// MorphNodeTo - This *mutates* the specified node to have the specified 7108 /// return type, opcode, and operands. 7109 /// 7110 /// Note that MorphNodeTo returns the resultant node. If there is already a 7111 /// node of the specified opcode and operands, it returns that node instead of 7112 /// the current one. Note that the SDLoc need not be the same. 7113 /// 7114 /// Using MorphNodeTo is faster than creating a new node and swapping it in 7115 /// with ReplaceAllUsesWith both because it often avoids allocating a new 7116 /// node, and because it doesn't require CSE recalculation for any of 7117 /// the node's users. 7118 /// 7119 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG. 7120 /// As a consequence it isn't appropriate to use from within the DAG combiner or 7121 /// the legalizer which maintain worklists that would need to be updated when 7122 /// deleting things. 7123 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, 7124 SDVTList VTs, ArrayRef<SDValue> Ops) { 7125 // If an identical node already exists, use it. 7126 void *IP = nullptr; 7127 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { 7128 FoldingSetNodeID ID; 7129 AddNodeIDNode(ID, Opc, VTs, Ops); 7130 if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP)) 7131 return UpdateSDLocOnMergeSDNode(ON, SDLoc(N)); 7132 } 7133 7134 if (!RemoveNodeFromCSEMaps(N)) 7135 IP = nullptr; 7136 7137 // Start the morphing. 7138 N->NodeType = Opc; 7139 N->ValueList = VTs.VTs; 7140 N->NumValues = VTs.NumVTs; 7141 7142 // Clear the operands list, updating used nodes to remove this from their 7143 // use list. Keep track of any operands that become dead as a result. 7144 SmallPtrSet<SDNode*, 16> DeadNodeSet; 7145 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 7146 SDUse &Use = *I++; 7147 SDNode *Used = Use.getNode(); 7148 Use.set(SDValue()); 7149 if (Used->use_empty()) 7150 DeadNodeSet.insert(Used); 7151 } 7152 7153 // For MachineNode, initialize the memory references information. 7154 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) 7155 MN->setMemRefs(nullptr, nullptr); 7156 7157 // Swap for an appropriately sized array from the recycler. 7158 removeOperands(N); 7159 createOperands(N, Ops); 7160 7161 // Delete any nodes that are still dead after adding the uses for the 7162 // new operands. 7163 if (!DeadNodeSet.empty()) { 7164 SmallVector<SDNode *, 16> DeadNodes; 7165 for (SDNode *N : DeadNodeSet) 7166 if (N->use_empty()) 7167 DeadNodes.push_back(N); 7168 RemoveDeadNodes(DeadNodes); 7169 } 7170 7171 if (IP) 7172 CSEMap.InsertNode(N, IP); // Memoize the new node. 7173 return N; 7174 } 7175 7176 SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) { 7177 unsigned OrigOpc = Node->getOpcode(); 7178 unsigned NewOpc; 7179 bool IsUnary = false; 7180 bool IsTernary = false; 7181 switch (OrigOpc) { 7182 default: 7183 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!"); 7184 case ISD::STRICT_FADD: NewOpc = ISD::FADD; break; 7185 case ISD::STRICT_FSUB: NewOpc = ISD::FSUB; break; 7186 case ISD::STRICT_FMUL: NewOpc = ISD::FMUL; break; 7187 case ISD::STRICT_FDIV: NewOpc = ISD::FDIV; break; 7188 case ISD::STRICT_FREM: NewOpc = ISD::FREM; break; 7189 case ISD::STRICT_FMA: NewOpc = ISD::FMA; IsTernary = true; break; 7190 case ISD::STRICT_FSQRT: NewOpc = ISD::FSQRT; IsUnary = true; break; 7191 case ISD::STRICT_FPOW: NewOpc = ISD::FPOW; break; 7192 case ISD::STRICT_FPOWI: NewOpc = ISD::FPOWI; break; 7193 case ISD::STRICT_FSIN: NewOpc = ISD::FSIN; IsUnary = true; break; 7194 case ISD::STRICT_FCOS: NewOpc = ISD::FCOS; IsUnary = true; break; 7195 case ISD::STRICT_FEXP: NewOpc = ISD::FEXP; IsUnary = true; break; 7196 case ISD::STRICT_FEXP2: NewOpc = ISD::FEXP2; IsUnary = true; break; 7197 case ISD::STRICT_FLOG: NewOpc = ISD::FLOG; IsUnary = true; break; 7198 case ISD::STRICT_FLOG10: NewOpc = ISD::FLOG10; IsUnary = true; break; 7199 case ISD::STRICT_FLOG2: NewOpc = ISD::FLOG2; IsUnary = true; break; 7200 case ISD::STRICT_FRINT: NewOpc = ISD::FRINT; IsUnary = true; break; 7201 case ISD::STRICT_FNEARBYINT: 7202 NewOpc = ISD::FNEARBYINT; 7203 IsUnary = true; 7204 break; 7205 } 7206 7207 // We're taking this node out of the chain, so we need to re-link things. 7208 SDValue InputChain = Node->getOperand(0); 7209 SDValue OutputChain = SDValue(Node, 1); 7210 ReplaceAllUsesOfValueWith(OutputChain, InputChain); 7211 7212 SDVTList VTs = getVTList(Node->getOperand(1).getValueType()); 7213 SDNode *Res = nullptr; 7214 if (IsUnary) 7215 Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1) }); 7216 else if (IsTernary) 7217 Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1), 7218 Node->getOperand(2), 7219 Node->getOperand(3)}); 7220 else 7221 Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1), 7222 Node->getOperand(2) }); 7223 7224 // MorphNodeTo can operate in two ways: if an existing node with the 7225 // specified operands exists, it can just return it. Otherwise, it 7226 // updates the node in place to have the requested operands. 7227 if (Res == Node) { 7228 // If we updated the node in place, reset the node ID. To the isel, 7229 // this should be just like a newly allocated machine node. 7230 Res->setNodeId(-1); 7231 } else { 7232 ReplaceAllUsesWith(Node, Res); 7233 RemoveDeadNode(Node); 7234 } 7235 7236 return Res; 7237 } 7238 7239 /// getMachineNode - These are used for target selectors to create a new node 7240 /// with specified return type(s), MachineInstr opcode, and operands. 7241 /// 7242 /// Note that getMachineNode returns the resultant node. If there is already a 7243 /// node of the specified opcode and operands, it returns that node instead of 7244 /// the current one. 7245 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 7246 EVT VT) { 7247 SDVTList VTs = getVTList(VT); 7248 return getMachineNode(Opcode, dl, VTs, None); 7249 } 7250 7251 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 7252 EVT VT, SDValue Op1) { 7253 SDVTList VTs = getVTList(VT); 7254 SDValue Ops[] = { Op1 }; 7255 return getMachineNode(Opcode, dl, VTs, Ops); 7256 } 7257 7258 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 7259 EVT VT, SDValue Op1, SDValue Op2) { 7260 SDVTList VTs = getVTList(VT); 7261 SDValue Ops[] = { Op1, Op2 }; 7262 return getMachineNode(Opcode, dl, VTs, Ops); 7263 } 7264 7265 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 7266 EVT VT, SDValue Op1, SDValue Op2, 7267 SDValue Op3) { 7268 SDVTList VTs = getVTList(VT); 7269 SDValue Ops[] = { Op1, Op2, Op3 }; 7270 return getMachineNode(Opcode, dl, VTs, Ops); 7271 } 7272 7273 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 7274 EVT VT, ArrayRef<SDValue> Ops) { 7275 SDVTList VTs = getVTList(VT); 7276 return getMachineNode(Opcode, dl, VTs, Ops); 7277 } 7278 7279 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 7280 EVT VT1, EVT VT2, SDValue Op1, 7281 SDValue Op2) { 7282 SDVTList VTs = getVTList(VT1, VT2); 7283 SDValue Ops[] = { Op1, Op2 }; 7284 return getMachineNode(Opcode, dl, VTs, Ops); 7285 } 7286 7287 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 7288 EVT VT1, EVT VT2, SDValue Op1, 7289 SDValue Op2, SDValue Op3) { 7290 SDVTList VTs = getVTList(VT1, VT2); 7291 SDValue Ops[] = { Op1, Op2, Op3 }; 7292 return getMachineNode(Opcode, dl, VTs, Ops); 7293 } 7294 7295 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 7296 EVT VT1, EVT VT2, 7297 ArrayRef<SDValue> Ops) { 7298 SDVTList VTs = getVTList(VT1, VT2); 7299 return getMachineNode(Opcode, dl, VTs, Ops); 7300 } 7301 7302 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 7303 EVT VT1, EVT VT2, EVT VT3, 7304 SDValue Op1, SDValue Op2) { 7305 SDVTList VTs = getVTList(VT1, VT2, VT3); 7306 SDValue Ops[] = { Op1, Op2 }; 7307 return getMachineNode(Opcode, dl, VTs, Ops); 7308 } 7309 7310 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 7311 EVT VT1, EVT VT2, EVT VT3, 7312 SDValue Op1, SDValue Op2, 7313 SDValue Op3) { 7314 SDVTList VTs = getVTList(VT1, VT2, VT3); 7315 SDValue Ops[] = { Op1, Op2, Op3 }; 7316 return getMachineNode(Opcode, dl, VTs, Ops); 7317 } 7318 7319 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 7320 EVT VT1, EVT VT2, EVT VT3, 7321 ArrayRef<SDValue> Ops) { 7322 SDVTList VTs = getVTList(VT1, VT2, VT3); 7323 return getMachineNode(Opcode, dl, VTs, Ops); 7324 } 7325 7326 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, 7327 ArrayRef<EVT> ResultTys, 7328 ArrayRef<SDValue> Ops) { 7329 SDVTList VTs = getVTList(ResultTys); 7330 return getMachineNode(Opcode, dl, VTs, Ops); 7331 } 7332 7333 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL, 7334 SDVTList VTs, 7335 ArrayRef<SDValue> Ops) { 7336 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; 7337 MachineSDNode *N; 7338 void *IP = nullptr; 7339 7340 if (DoCSE) { 7341 FoldingSetNodeID ID; 7342 AddNodeIDNode(ID, ~Opcode, VTs, Ops); 7343 IP = nullptr; 7344 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { 7345 return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL)); 7346 } 7347 } 7348 7349 // Allocate a new MachineSDNode. 7350 N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); 7351 createOperands(N, Ops); 7352 7353 if (DoCSE) 7354 CSEMap.InsertNode(N, IP); 7355 7356 InsertNode(N); 7357 return N; 7358 } 7359 7360 /// getTargetExtractSubreg - A convenience function for creating 7361 /// TargetOpcode::EXTRACT_SUBREG nodes. 7362 SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, 7363 SDValue Operand) { 7364 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32); 7365 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 7366 VT, Operand, SRIdxVal); 7367 return SDValue(Subreg, 0); 7368 } 7369 7370 /// getTargetInsertSubreg - A convenience function for creating 7371 /// TargetOpcode::INSERT_SUBREG nodes. 7372 SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, 7373 SDValue Operand, SDValue Subreg) { 7374 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32); 7375 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 7376 VT, Operand, Subreg, SRIdxVal); 7377 return SDValue(Result, 0); 7378 } 7379 7380 /// getNodeIfExists - Get the specified node if it's already available, or 7381 /// else return NULL. 7382 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, 7383 ArrayRef<SDValue> Ops, 7384 const SDNodeFlags Flags) { 7385 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { 7386 FoldingSetNodeID ID; 7387 AddNodeIDNode(ID, Opcode, VTList, Ops); 7388 void *IP = nullptr; 7389 if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) { 7390 E->intersectFlagsWith(Flags); 7391 return E; 7392 } 7393 } 7394 return nullptr; 7395 } 7396 7397 /// getDbgValue - Creates a SDDbgValue node. 7398 /// 7399 /// SDNode 7400 SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr, 7401 SDNode *N, unsigned R, bool IsIndirect, 7402 const DebugLoc &DL, unsigned O) { 7403 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && 7404 "Expected inlined-at fields to agree"); 7405 return new (DbgInfo->getAlloc()) 7406 SDDbgValue(Var, Expr, N, R, IsIndirect, DL, O); 7407 } 7408 7409 /// Constant 7410 SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var, 7411 DIExpression *Expr, 7412 const Value *C, 7413 const DebugLoc &DL, unsigned O) { 7414 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && 7415 "Expected inlined-at fields to agree"); 7416 return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, C, DL, O); 7417 } 7418 7419 /// FrameIndex 7420 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var, 7421 DIExpression *Expr, unsigned FI, 7422 bool IsIndirect, 7423 const DebugLoc &DL, 7424 unsigned O) { 7425 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && 7426 "Expected inlined-at fields to agree"); 7427 return new (DbgInfo->getAlloc()) 7428 SDDbgValue(Var, Expr, FI, IsIndirect, DL, O, SDDbgValue::FRAMEIX); 7429 } 7430 7431 /// VReg 7432 SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, 7433 DIExpression *Expr, 7434 unsigned VReg, bool IsIndirect, 7435 const DebugLoc &DL, unsigned O) { 7436 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && 7437 "Expected inlined-at fields to agree"); 7438 return new (DbgInfo->getAlloc()) 7439 SDDbgValue(Var, Expr, VReg, IsIndirect, DL, O, SDDbgValue::VREG); 7440 } 7441 7442 void SelectionDAG::transferDbgValues(SDValue From, SDValue To, 7443 unsigned OffsetInBits, unsigned SizeInBits, 7444 bool InvalidateDbg) { 7445 SDNode *FromNode = From.getNode(); 7446 SDNode *ToNode = To.getNode(); 7447 assert(FromNode && ToNode && "Can't modify dbg values"); 7448 7449 // PR35338 7450 // TODO: assert(From != To && "Redundant dbg value transfer"); 7451 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer"); 7452 if (From == To || FromNode == ToNode) 7453 return; 7454 7455 if (!FromNode->getHasDebugValue()) 7456 return; 7457 7458 SmallVector<SDDbgValue *, 2> ClonedDVs; 7459 for (SDDbgValue *Dbg : GetDbgValues(FromNode)) { 7460 if (Dbg->getKind() != SDDbgValue::SDNODE || Dbg->isInvalidated()) 7461 continue; 7462 7463 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value"); 7464 7465 // Just transfer the dbg value attached to From. 7466 if (Dbg->getResNo() != From.getResNo()) 7467 continue; 7468 7469 DIVariable *Var = Dbg->getVariable(); 7470 auto *Expr = Dbg->getExpression(); 7471 // If a fragment is requested, update the expression. 7472 if (SizeInBits) { 7473 // When splitting a larger (e.g., sign-extended) value whose 7474 // lower bits are described with an SDDbgValue, do not attempt 7475 // to transfer the SDDbgValue to the upper bits. 7476 if (auto FI = Expr->getFragmentInfo()) 7477 if (OffsetInBits + SizeInBits > FI->SizeInBits) 7478 continue; 7479 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits, 7480 SizeInBits); 7481 if (!Fragment) 7482 continue; 7483 Expr = *Fragment; 7484 } 7485 // Clone the SDDbgValue and move it to To. 7486 SDDbgValue *Clone = 7487 getDbgValue(Var, Expr, ToNode, To.getResNo(), Dbg->isIndirect(), 7488 Dbg->getDebugLoc(), Dbg->getOrder()); 7489 ClonedDVs.push_back(Clone); 7490 7491 if (InvalidateDbg) 7492 Dbg->setIsInvalidated(); 7493 } 7494 7495 for (SDDbgValue *Dbg : ClonedDVs) 7496 AddDbgValue(Dbg, ToNode, false); 7497 } 7498 7499 void SelectionDAG::salvageDebugInfo(SDNode &N) { 7500 if (!N.getHasDebugValue()) 7501 return; 7502 7503 SmallVector<SDDbgValue *, 2> ClonedDVs; 7504 for (auto DV : GetDbgValues(&N)) { 7505 if (DV->isInvalidated()) 7506 continue; 7507 switch (N.getOpcode()) { 7508 default: 7509 break; 7510 case ISD::ADD: 7511 SDValue N0 = N.getOperand(0); 7512 SDValue N1 = N.getOperand(1); 7513 if (!isConstantIntBuildVectorOrConstantInt(N0) && 7514 isConstantIntBuildVectorOrConstantInt(N1)) { 7515 uint64_t Offset = N.getConstantOperandVal(1); 7516 // Rewrite an ADD constant node into a DIExpression. Since we are 7517 // performing arithmetic to compute the variable's *value* in the 7518 // DIExpression, we need to mark the expression with a 7519 // DW_OP_stack_value. 7520 auto *DIExpr = DV->getExpression(); 7521 DIExpr = DIExpression::prepend(DIExpr, DIExpression::NoDeref, Offset, 7522 DIExpression::NoDeref, 7523 DIExpression::WithStackValue); 7524 SDDbgValue *Clone = 7525 getDbgValue(DV->getVariable(), DIExpr, N0.getNode(), N0.getResNo(), 7526 DV->isIndirect(), DV->getDebugLoc(), DV->getOrder()); 7527 ClonedDVs.push_back(Clone); 7528 DV->setIsInvalidated(); 7529 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; 7530 N0.getNode()->dumprFull(this); 7531 dbgs() << " into " << *DIExpr << '\n'); 7532 } 7533 } 7534 } 7535 7536 for (SDDbgValue *Dbg : ClonedDVs) 7537 AddDbgValue(Dbg, Dbg->getSDNode(), false); 7538 } 7539 7540 /// Creates a SDDbgLabel node. 7541 SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label, 7542 const DebugLoc &DL, unsigned O) { 7543 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) && 7544 "Expected inlined-at fields to agree"); 7545 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O); 7546 } 7547 7548 namespace { 7549 7550 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node 7551 /// pointed to by a use iterator is deleted, increment the use iterator 7552 /// so that it doesn't dangle. 7553 /// 7554 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { 7555 SDNode::use_iterator &UI; 7556 SDNode::use_iterator &UE; 7557 7558 void NodeDeleted(SDNode *N, SDNode *E) override { 7559 // Increment the iterator as needed. 7560 while (UI != UE && N == *UI) 7561 ++UI; 7562 } 7563 7564 public: 7565 RAUWUpdateListener(SelectionDAG &d, 7566 SDNode::use_iterator &ui, 7567 SDNode::use_iterator &ue) 7568 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {} 7569 }; 7570 7571 } // end anonymous namespace 7572 7573 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 7574 /// This can cause recursive merging of nodes in the DAG. 7575 /// 7576 /// This version assumes From has a single result value. 7577 /// 7578 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) { 7579 SDNode *From = FromN.getNode(); 7580 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && 7581 "Cannot replace with this method!"); 7582 assert(From != To.getNode() && "Cannot replace uses of with self"); 7583 7584 // Preserve Debug Values 7585 transferDbgValues(FromN, To); 7586 7587 // Iterate over all the existing uses of From. New uses will be added 7588 // to the beginning of the use list, which we avoid visiting. 7589 // This specifically avoids visiting uses of From that arise while the 7590 // replacement is happening, because any such uses would be the result 7591 // of CSE: If an existing node looks like From after one of its operands 7592 // is replaced by To, we don't want to replace of all its users with To 7593 // too. See PR3018 for more info. 7594 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 7595 RAUWUpdateListener Listener(*this, UI, UE); 7596 while (UI != UE) { 7597 SDNode *User = *UI; 7598 7599 // This node is about to morph, remove its old self from the CSE maps. 7600 RemoveNodeFromCSEMaps(User); 7601 7602 // A user can appear in a use list multiple times, and when this 7603 // happens the uses are usually next to each other in the list. 7604 // To help reduce the number of CSE recomputations, process all 7605 // the uses of this user that we can find this way. 7606 do { 7607 SDUse &Use = UI.getUse(); 7608 ++UI; 7609 Use.set(To); 7610 if (To->isDivergent() != From->isDivergent()) 7611 updateDivergence(User); 7612 } while (UI != UE && *UI == User); 7613 // Now that we have modified User, add it back to the CSE maps. If it 7614 // already exists there, recursively merge the results together. 7615 AddModifiedNodeToCSEMaps(User); 7616 } 7617 7618 // If we just RAUW'd the root, take note. 7619 if (FromN == getRoot()) 7620 setRoot(To); 7621 } 7622 7623 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 7624 /// This can cause recursive merging of nodes in the DAG. 7625 /// 7626 /// This version assumes that for each value of From, there is a 7627 /// corresponding value in To in the same position with the same type. 7628 /// 7629 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) { 7630 #ifndef NDEBUG 7631 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 7632 assert((!From->hasAnyUseOfValue(i) || 7633 From->getValueType(i) == To->getValueType(i)) && 7634 "Cannot use this version of ReplaceAllUsesWith!"); 7635 #endif 7636 7637 // Handle the trivial case. 7638 if (From == To) 7639 return; 7640 7641 // Preserve Debug Info. Only do this if there's a use. 7642 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 7643 if (From->hasAnyUseOfValue(i)) { 7644 assert((i < To->getNumValues()) && "Invalid To location"); 7645 transferDbgValues(SDValue(From, i), SDValue(To, i)); 7646 } 7647 7648 // Iterate over just the existing users of From. See the comments in 7649 // the ReplaceAllUsesWith above. 7650 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 7651 RAUWUpdateListener Listener(*this, UI, UE); 7652 while (UI != UE) { 7653 SDNode *User = *UI; 7654 7655 // This node is about to morph, remove its old self from the CSE maps. 7656 RemoveNodeFromCSEMaps(User); 7657 7658 // A user can appear in a use list multiple times, and when this 7659 // happens the uses are usually next to each other in the list. 7660 // To help reduce the number of CSE recomputations, process all 7661 // the uses of this user that we can find this way. 7662 do { 7663 SDUse &Use = UI.getUse(); 7664 ++UI; 7665 Use.setNode(To); 7666 if (To->isDivergent() != From->isDivergent()) 7667 updateDivergence(User); 7668 } while (UI != UE && *UI == User); 7669 7670 // Now that we have modified User, add it back to the CSE maps. If it 7671 // already exists there, recursively merge the results together. 7672 AddModifiedNodeToCSEMaps(User); 7673 } 7674 7675 // If we just RAUW'd the root, take note. 7676 if (From == getRoot().getNode()) 7677 setRoot(SDValue(To, getRoot().getResNo())); 7678 } 7679 7680 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 7681 /// This can cause recursive merging of nodes in the DAG. 7682 /// 7683 /// This version can replace From with any result values. To must match the 7684 /// number and types of values returned by From. 7685 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) { 7686 if (From->getNumValues() == 1) // Handle the simple case efficiently. 7687 return ReplaceAllUsesWith(SDValue(From, 0), To[0]); 7688 7689 // Preserve Debug Info. 7690 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 7691 transferDbgValues(SDValue(From, i), *To); 7692 7693 // Iterate over just the existing users of From. See the comments in 7694 // the ReplaceAllUsesWith above. 7695 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 7696 RAUWUpdateListener Listener(*this, UI, UE); 7697 while (UI != UE) { 7698 SDNode *User = *UI; 7699 7700 // This node is about to morph, remove its old self from the CSE maps. 7701 RemoveNodeFromCSEMaps(User); 7702 7703 // A user can appear in a use list multiple times, and when this 7704 // happens the uses are usually next to each other in the list. 7705 // To help reduce the number of CSE recomputations, process all 7706 // the uses of this user that we can find this way. 7707 do { 7708 SDUse &Use = UI.getUse(); 7709 const SDValue &ToOp = To[Use.getResNo()]; 7710 ++UI; 7711 Use.set(ToOp); 7712 if (To->getNode()->isDivergent() != From->isDivergent()) 7713 updateDivergence(User); 7714 } while (UI != UE && *UI == User); 7715 // Now that we have modified User, add it back to the CSE maps. If it 7716 // already exists there, recursively merge the results together. 7717 AddModifiedNodeToCSEMaps(User); 7718 } 7719 7720 // If we just RAUW'd the root, take note. 7721 if (From == getRoot().getNode()) 7722 setRoot(SDValue(To[getRoot().getResNo()])); 7723 } 7724 7725 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving 7726 /// uses of other values produced by From.getNode() alone. The Deleted 7727 /// vector is handled the same way as for ReplaceAllUsesWith. 7728 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){ 7729 // Handle the really simple, really trivial case efficiently. 7730 if (From == To) return; 7731 7732 // Handle the simple, trivial, case efficiently. 7733 if (From.getNode()->getNumValues() == 1) { 7734 ReplaceAllUsesWith(From, To); 7735 return; 7736 } 7737 7738 // Preserve Debug Info. 7739 transferDbgValues(From, To); 7740 7741 // Iterate over just the existing users of From. See the comments in 7742 // the ReplaceAllUsesWith above. 7743 SDNode::use_iterator UI = From.getNode()->use_begin(), 7744 UE = From.getNode()->use_end(); 7745 RAUWUpdateListener Listener(*this, UI, UE); 7746 while (UI != UE) { 7747 SDNode *User = *UI; 7748 bool UserRemovedFromCSEMaps = false; 7749 7750 // A user can appear in a use list multiple times, and when this 7751 // happens the uses are usually next to each other in the list. 7752 // To help reduce the number of CSE recomputations, process all 7753 // the uses of this user that we can find this way. 7754 do { 7755 SDUse &Use = UI.getUse(); 7756 7757 // Skip uses of different values from the same node. 7758 if (Use.getResNo() != From.getResNo()) { 7759 ++UI; 7760 continue; 7761 } 7762 7763 // If this node hasn't been modified yet, it's still in the CSE maps, 7764 // so remove its old self from the CSE maps. 7765 if (!UserRemovedFromCSEMaps) { 7766 RemoveNodeFromCSEMaps(User); 7767 UserRemovedFromCSEMaps = true; 7768 } 7769 7770 ++UI; 7771 Use.set(To); 7772 if (To->isDivergent() != From->isDivergent()) 7773 updateDivergence(User); 7774 } while (UI != UE && *UI == User); 7775 // We are iterating over all uses of the From node, so if a use 7776 // doesn't use the specific value, no changes are made. 7777 if (!UserRemovedFromCSEMaps) 7778 continue; 7779 7780 // Now that we have modified User, add it back to the CSE maps. If it 7781 // already exists there, recursively merge the results together. 7782 AddModifiedNodeToCSEMaps(User); 7783 } 7784 7785 // If we just RAUW'd the root, take note. 7786 if (From == getRoot()) 7787 setRoot(To); 7788 } 7789 7790 namespace { 7791 7792 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith 7793 /// to record information about a use. 7794 struct UseMemo { 7795 SDNode *User; 7796 unsigned Index; 7797 SDUse *Use; 7798 }; 7799 7800 /// operator< - Sort Memos by User. 7801 bool operator<(const UseMemo &L, const UseMemo &R) { 7802 return (intptr_t)L.User < (intptr_t)R.User; 7803 } 7804 7805 } // end anonymous namespace 7806 7807 void SelectionDAG::updateDivergence(SDNode * N) 7808 { 7809 if (TLI->isSDNodeAlwaysUniform(N)) 7810 return; 7811 bool IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, DA); 7812 for (auto &Op : N->ops()) { 7813 if (Op.Val.getValueType() != MVT::Other) 7814 IsDivergent |= Op.getNode()->isDivergent(); 7815 } 7816 if (N->SDNodeBits.IsDivergent != IsDivergent) { 7817 N->SDNodeBits.IsDivergent = IsDivergent; 7818 for (auto U : N->uses()) { 7819 updateDivergence(U); 7820 } 7821 } 7822 } 7823 7824 7825 void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode*>& Order) { 7826 DenseMap<SDNode *, unsigned> Degree; 7827 Order.reserve(AllNodes.size()); 7828 for (auto & N : allnodes()) { 7829 unsigned NOps = N.getNumOperands(); 7830 Degree[&N] = NOps; 7831 if (0 == NOps) 7832 Order.push_back(&N); 7833 } 7834 for (std::vector<SDNode *>::iterator I = Order.begin(); 7835 I!=Order.end();++I) { 7836 SDNode * N = *I; 7837 for (auto U : N->uses()) { 7838 unsigned &UnsortedOps = Degree[U]; 7839 if (0 == --UnsortedOps) 7840 Order.push_back(U); 7841 } 7842 } 7843 } 7844 7845 void SelectionDAG::VerifyDAGDiverence() 7846 { 7847 std::vector<SDNode*> TopoOrder; 7848 CreateTopologicalOrder(TopoOrder); 7849 const TargetLowering &TLI = getTargetLoweringInfo(); 7850 DenseMap<const SDNode *, bool> DivergenceMap; 7851 for (auto &N : allnodes()) { 7852 DivergenceMap[&N] = false; 7853 } 7854 for (auto N : TopoOrder) { 7855 bool IsDivergent = DivergenceMap[N]; 7856 bool IsSDNodeDivergent = TLI.isSDNodeSourceOfDivergence(N, FLI, DA); 7857 for (auto &Op : N->ops()) { 7858 if (Op.Val.getValueType() != MVT::Other) 7859 IsSDNodeDivergent |= DivergenceMap[Op.getNode()]; 7860 } 7861 if (!IsDivergent && IsSDNodeDivergent && !TLI.isSDNodeAlwaysUniform(N)) { 7862 DivergenceMap[N] = true; 7863 } 7864 } 7865 for (auto &N : allnodes()) { 7866 (void)N; 7867 assert(DivergenceMap[&N] == N.isDivergent() && 7868 "Divergence bit inconsistency detected\n"); 7869 } 7870 } 7871 7872 7873 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving 7874 /// uses of other values produced by From.getNode() alone. The same value 7875 /// may appear in both the From and To list. The Deleted vector is 7876 /// handled the same way as for ReplaceAllUsesWith. 7877 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, 7878 const SDValue *To, 7879 unsigned Num){ 7880 // Handle the simple, trivial case efficiently. 7881 if (Num == 1) 7882 return ReplaceAllUsesOfValueWith(*From, *To); 7883 7884 transferDbgValues(*From, *To); 7885 7886 // Read up all the uses and make records of them. This helps 7887 // processing new uses that are introduced during the 7888 // replacement process. 7889 SmallVector<UseMemo, 4> Uses; 7890 for (unsigned i = 0; i != Num; ++i) { 7891 unsigned FromResNo = From[i].getResNo(); 7892 SDNode *FromNode = From[i].getNode(); 7893 for (SDNode::use_iterator UI = FromNode->use_begin(), 7894 E = FromNode->use_end(); UI != E; ++UI) { 7895 SDUse &Use = UI.getUse(); 7896 if (Use.getResNo() == FromResNo) { 7897 UseMemo Memo = { *UI, i, &Use }; 7898 Uses.push_back(Memo); 7899 } 7900 } 7901 } 7902 7903 // Sort the uses, so that all the uses from a given User are together. 7904 llvm::sort(Uses.begin(), Uses.end()); 7905 7906 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); 7907 UseIndex != UseIndexEnd; ) { 7908 // We know that this user uses some value of From. If it is the right 7909 // value, update it. 7910 SDNode *User = Uses[UseIndex].User; 7911 7912 // This node is about to morph, remove its old self from the CSE maps. 7913 RemoveNodeFromCSEMaps(User); 7914 7915 // The Uses array is sorted, so all the uses for a given User 7916 // are next to each other in the list. 7917 // To help reduce the number of CSE recomputations, process all 7918 // the uses of this user that we can find this way. 7919 do { 7920 unsigned i = Uses[UseIndex].Index; 7921 SDUse &Use = *Uses[UseIndex].Use; 7922 ++UseIndex; 7923 7924 Use.set(To[i]); 7925 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User); 7926 7927 // Now that we have modified User, add it back to the CSE maps. If it 7928 // already exists there, recursively merge the results together. 7929 AddModifiedNodeToCSEMaps(User); 7930 } 7931 } 7932 7933 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG 7934 /// based on their topological order. It returns the maximum id and a vector 7935 /// of the SDNodes* in assigned order by reference. 7936 unsigned SelectionDAG::AssignTopologicalOrder() { 7937 unsigned DAGSize = 0; 7938 7939 // SortedPos tracks the progress of the algorithm. Nodes before it are 7940 // sorted, nodes after it are unsorted. When the algorithm completes 7941 // it is at the end of the list. 7942 allnodes_iterator SortedPos = allnodes_begin(); 7943 7944 // Visit all the nodes. Move nodes with no operands to the front of 7945 // the list immediately. Annotate nodes that do have operands with their 7946 // operand count. Before we do this, the Node Id fields of the nodes 7947 // may contain arbitrary values. After, the Node Id fields for nodes 7948 // before SortedPos will contain the topological sort index, and the 7949 // Node Id fields for nodes At SortedPos and after will contain the 7950 // count of outstanding operands. 7951 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) { 7952 SDNode *N = &*I++; 7953 checkForCycles(N, this); 7954 unsigned Degree = N->getNumOperands(); 7955 if (Degree == 0) { 7956 // A node with no uses, add it to the result array immediately. 7957 N->setNodeId(DAGSize++); 7958 allnodes_iterator Q(N); 7959 if (Q != SortedPos) 7960 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); 7961 assert(SortedPos != AllNodes.end() && "Overran node list"); 7962 ++SortedPos; 7963 } else { 7964 // Temporarily use the Node Id as scratch space for the degree count. 7965 N->setNodeId(Degree); 7966 } 7967 } 7968 7969 // Visit all the nodes. As we iterate, move nodes into sorted order, 7970 // such that by the time the end is reached all nodes will be sorted. 7971 for (SDNode &Node : allnodes()) { 7972 SDNode *N = &Node; 7973 checkForCycles(N, this); 7974 // N is in sorted position, so all its uses have one less operand 7975 // that needs to be sorted. 7976 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 7977 UI != UE; ++UI) { 7978 SDNode *P = *UI; 7979 unsigned Degree = P->getNodeId(); 7980 assert(Degree != 0 && "Invalid node degree"); 7981 --Degree; 7982 if (Degree == 0) { 7983 // All of P's operands are sorted, so P may sorted now. 7984 P->setNodeId(DAGSize++); 7985 if (P->getIterator() != SortedPos) 7986 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); 7987 assert(SortedPos != AllNodes.end() && "Overran node list"); 7988 ++SortedPos; 7989 } else { 7990 // Update P's outstanding operand count. 7991 P->setNodeId(Degree); 7992 } 7993 } 7994 if (Node.getIterator() == SortedPos) { 7995 #ifndef NDEBUG 7996 allnodes_iterator I(N); 7997 SDNode *S = &*++I; 7998 dbgs() << "Overran sorted position:\n"; 7999 S->dumprFull(this); dbgs() << "\n"; 8000 dbgs() << "Checking if this is due to cycles\n"; 8001 checkForCycles(this, true); 8002 #endif 8003 llvm_unreachable(nullptr); 8004 } 8005 } 8006 8007 assert(SortedPos == AllNodes.end() && 8008 "Topological sort incomplete!"); 8009 assert(AllNodes.front().getOpcode() == ISD::EntryToken && 8010 "First node in topological sort is not the entry token!"); 8011 assert(AllNodes.front().getNodeId() == 0 && 8012 "First node in topological sort has non-zero id!"); 8013 assert(AllNodes.front().getNumOperands() == 0 && 8014 "First node in topological sort has operands!"); 8015 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && 8016 "Last node in topologic sort has unexpected id!"); 8017 assert(AllNodes.back().use_empty() && 8018 "Last node in topologic sort has users!"); 8019 assert(DAGSize == allnodes_size() && "Node count mismatch!"); 8020 return DAGSize; 8021 } 8022 8023 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the 8024 /// value is produced by SD. 8025 void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) { 8026 if (SD) { 8027 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue()); 8028 SD->setHasDebugValue(true); 8029 } 8030 DbgInfo->add(DB, SD, isParameter); 8031 } 8032 8033 void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { 8034 DbgInfo->add(DB); 8035 } 8036 8037 SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, 8038 SDValue NewMemOp) { 8039 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node"); 8040 // The new memory operation must have the same position as the old load in 8041 // terms of memory dependency. Create a TokenFactor for the old load and new 8042 // memory operation and update uses of the old load's output chain to use that 8043 // TokenFactor. 8044 SDValue OldChain = SDValue(OldLoad, 1); 8045 SDValue NewChain = SDValue(NewMemOp.getNode(), 1); 8046 if (!OldLoad->hasAnyUseOfValue(1)) 8047 return NewChain; 8048 8049 SDValue TokenFactor = 8050 getNode(ISD::TokenFactor, SDLoc(OldLoad), MVT::Other, OldChain, NewChain); 8051 ReplaceAllUsesOfValueWith(OldChain, TokenFactor); 8052 UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewChain); 8053 return TokenFactor; 8054 } 8055 8056 //===----------------------------------------------------------------------===// 8057 // SDNode Class 8058 //===----------------------------------------------------------------------===// 8059 8060 bool llvm::isNullConstant(SDValue V) { 8061 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); 8062 return Const != nullptr && Const->isNullValue(); 8063 } 8064 8065 bool llvm::isNullFPConstant(SDValue V) { 8066 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V); 8067 return Const != nullptr && Const->isZero() && !Const->isNegative(); 8068 } 8069 8070 bool llvm::isAllOnesConstant(SDValue V) { 8071 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); 8072 return Const != nullptr && Const->isAllOnesValue(); 8073 } 8074 8075 bool llvm::isOneConstant(SDValue V) { 8076 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); 8077 return Const != nullptr && Const->isOne(); 8078 } 8079 8080 bool llvm::isBitwiseNot(SDValue V) { 8081 return V.getOpcode() == ISD::XOR && isAllOnesConstant(V.getOperand(1)); 8082 } 8083 8084 ConstantSDNode *llvm::isConstOrConstSplat(SDValue N) { 8085 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) 8086 return CN; 8087 8088 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) { 8089 BitVector UndefElements; 8090 ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements); 8091 8092 // BuildVectors can truncate their operands. Ignore that case here. 8093 // FIXME: We blindly ignore splats which include undef which is overly 8094 // pessimistic. 8095 if (CN && UndefElements.none() && 8096 CN->getValueType(0) == N.getValueType().getScalarType()) 8097 return CN; 8098 } 8099 8100 return nullptr; 8101 } 8102 8103 ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N) { 8104 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N)) 8105 return CN; 8106 8107 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) { 8108 BitVector UndefElements; 8109 ConstantFPSDNode *CN = BV->getConstantFPSplatNode(&UndefElements); 8110 8111 if (CN && UndefElements.none()) 8112 return CN; 8113 } 8114 8115 return nullptr; 8116 } 8117 8118 HandleSDNode::~HandleSDNode() { 8119 DropOperands(); 8120 } 8121 8122 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order, 8123 const DebugLoc &DL, 8124 const GlobalValue *GA, EVT VT, 8125 int64_t o, unsigned char TF) 8126 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { 8127 TheGlobal = GA; 8128 } 8129 8130 AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, 8131 EVT VT, unsigned SrcAS, 8132 unsigned DestAS) 8133 : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)), 8134 SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {} 8135 8136 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, 8137 SDVTList VTs, EVT memvt, MachineMemOperand *mmo) 8138 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { 8139 MemSDNodeBits.IsVolatile = MMO->isVolatile(); 8140 MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal(); 8141 MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable(); 8142 MemSDNodeBits.IsInvariant = MMO->isInvariant(); 8143 8144 // We check here that the size of the memory operand fits within the size of 8145 // the MMO. This is because the MMO might indicate only a possible address 8146 // range instead of specifying the affected memory addresses precisely. 8147 assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!"); 8148 } 8149 8150 /// Profile - Gather unique data for the node. 8151 /// 8152 void SDNode::Profile(FoldingSetNodeID &ID) const { 8153 AddNodeIDNode(ID, this); 8154 } 8155 8156 namespace { 8157 8158 struct EVTArray { 8159 std::vector<EVT> VTs; 8160 8161 EVTArray() { 8162 VTs.reserve(MVT::LAST_VALUETYPE); 8163 for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i) 8164 VTs.push_back(MVT((MVT::SimpleValueType)i)); 8165 } 8166 }; 8167 8168 } // end anonymous namespace 8169 8170 static ManagedStatic<std::set<EVT, EVT::compareRawBits>> EVTs; 8171 static ManagedStatic<EVTArray> SimpleVTArray; 8172 static ManagedStatic<sys::SmartMutex<true>> VTMutex; 8173 8174 /// getValueTypeList - Return a pointer to the specified value type. 8175 /// 8176 const EVT *SDNode::getValueTypeList(EVT VT) { 8177 if (VT.isExtended()) { 8178 sys::SmartScopedLock<true> Lock(*VTMutex); 8179 return &(*EVTs->insert(VT).first); 8180 } else { 8181 assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE && 8182 "Value type out of range!"); 8183 return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy]; 8184 } 8185 } 8186 8187 /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the 8188 /// indicated value. This method ignores uses of other values defined by this 8189 /// operation. 8190 bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { 8191 assert(Value < getNumValues() && "Bad value!"); 8192 8193 // TODO: Only iterate over uses of a given value of the node 8194 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { 8195 if (UI.getUse().getResNo() == Value) { 8196 if (NUses == 0) 8197 return false; 8198 --NUses; 8199 } 8200 } 8201 8202 // Found exactly the right number of uses? 8203 return NUses == 0; 8204 } 8205 8206 /// hasAnyUseOfValue - Return true if there are any use of the indicated 8207 /// value. This method ignores uses of other values defined by this operation. 8208 bool SDNode::hasAnyUseOfValue(unsigned Value) const { 8209 assert(Value < getNumValues() && "Bad value!"); 8210 8211 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) 8212 if (UI.getUse().getResNo() == Value) 8213 return true; 8214 8215 return false; 8216 } 8217 8218 /// isOnlyUserOf - Return true if this node is the only use of N. 8219 bool SDNode::isOnlyUserOf(const SDNode *N) const { 8220 bool Seen = false; 8221 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 8222 SDNode *User = *I; 8223 if (User == this) 8224 Seen = true; 8225 else 8226 return false; 8227 } 8228 8229 return Seen; 8230 } 8231 8232 /// Return true if the only users of N are contained in Nodes. 8233 bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) { 8234 bool Seen = false; 8235 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 8236 SDNode *User = *I; 8237 if (llvm::any_of(Nodes, 8238 [&User](const SDNode *Node) { return User == Node; })) 8239 Seen = true; 8240 else 8241 return false; 8242 } 8243 8244 return Seen; 8245 } 8246 8247 /// isOperand - Return true if this node is an operand of N. 8248 bool SDValue::isOperandOf(const SDNode *N) const { 8249 for (const SDValue &Op : N->op_values()) 8250 if (*this == Op) 8251 return true; 8252 return false; 8253 } 8254 8255 bool SDNode::isOperandOf(const SDNode *N) const { 8256 for (const SDValue &Op : N->op_values()) 8257 if (this == Op.getNode()) 8258 return true; 8259 return false; 8260 } 8261 8262 /// reachesChainWithoutSideEffects - Return true if this operand (which must 8263 /// be a chain) reaches the specified operand without crossing any 8264 /// side-effecting instructions on any chain path. In practice, this looks 8265 /// through token factors and non-volatile loads. In order to remain efficient, 8266 /// this only looks a couple of nodes in, it does not do an exhaustive search. 8267 /// 8268 /// Note that we only need to examine chains when we're searching for 8269 /// side-effects; SelectionDAG requires that all side-effects are represented 8270 /// by chains, even if another operand would force a specific ordering. This 8271 /// constraint is necessary to allow transformations like splitting loads. 8272 bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, 8273 unsigned Depth) const { 8274 if (*this == Dest) return true; 8275 8276 // Don't search too deeply, we just want to be able to see through 8277 // TokenFactor's etc. 8278 if (Depth == 0) return false; 8279 8280 // If this is a token factor, all inputs to the TF happen in parallel. 8281 if (getOpcode() == ISD::TokenFactor) { 8282 // First, try a shallow search. 8283 if (is_contained((*this)->ops(), Dest)) { 8284 // We found the chain we want as an operand of this TokenFactor. 8285 // Essentially, we reach the chain without side-effects if we could 8286 // serialize the TokenFactor into a simple chain of operations with 8287 // Dest as the last operation. This is automatically true if the 8288 // chain has one use: there are no other ordering constraints. 8289 // If the chain has more than one use, we give up: some other 8290 // use of Dest might force a side-effect between Dest and the current 8291 // node. 8292 if (Dest.hasOneUse()) 8293 return true; 8294 } 8295 // Next, try a deep search: check whether every operand of the TokenFactor 8296 // reaches Dest. 8297 return llvm::all_of((*this)->ops(), [=](SDValue Op) { 8298 return Op.reachesChainWithoutSideEffects(Dest, Depth - 1); 8299 }); 8300 } 8301 8302 // Loads don't have side effects, look through them. 8303 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) { 8304 if (!Ld->isVolatile()) 8305 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); 8306 } 8307 return false; 8308 } 8309 8310 bool SDNode::hasPredecessor(const SDNode *N) const { 8311 SmallPtrSet<const SDNode *, 32> Visited; 8312 SmallVector<const SDNode *, 16> Worklist; 8313 Worklist.push_back(this); 8314 return hasPredecessorHelper(N, Visited, Worklist); 8315 } 8316 8317 void SDNode::intersectFlagsWith(const SDNodeFlags Flags) { 8318 this->Flags.intersectWith(Flags); 8319 } 8320 8321 SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { 8322 assert(N->getNumValues() == 1 && 8323 "Can't unroll a vector with multiple results!"); 8324 8325 EVT VT = N->getValueType(0); 8326 unsigned NE = VT.getVectorNumElements(); 8327 EVT EltVT = VT.getVectorElementType(); 8328 SDLoc dl(N); 8329 8330 SmallVector<SDValue, 8> Scalars; 8331 SmallVector<SDValue, 4> Operands(N->getNumOperands()); 8332 8333 // If ResNE is 0, fully unroll the vector op. 8334 if (ResNE == 0) 8335 ResNE = NE; 8336 else if (NE > ResNE) 8337 NE = ResNE; 8338 8339 unsigned i; 8340 for (i= 0; i != NE; ++i) { 8341 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { 8342 SDValue Operand = N->getOperand(j); 8343 EVT OperandVT = Operand.getValueType(); 8344 if (OperandVT.isVector()) { 8345 // A vector operand; extract a single element. 8346 EVT OperandEltVT = OperandVT.getVectorElementType(); 8347 Operands[j] = 8348 getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand, 8349 getConstant(i, dl, TLI->getVectorIdxTy(getDataLayout()))); 8350 } else { 8351 // A scalar operand; just use it as is. 8352 Operands[j] = Operand; 8353 } 8354 } 8355 8356 switch (N->getOpcode()) { 8357 default: { 8358 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands, 8359 N->getFlags())); 8360 break; 8361 } 8362 case ISD::VSELECT: 8363 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands)); 8364 break; 8365 case ISD::SHL: 8366 case ISD::SRA: 8367 case ISD::SRL: 8368 case ISD::ROTL: 8369 case ISD::ROTR: 8370 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], 8371 getShiftAmountOperand(Operands[0].getValueType(), 8372 Operands[1]))); 8373 break; 8374 case ISD::SIGN_EXTEND_INREG: 8375 case ISD::FP_ROUND_INREG: { 8376 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType(); 8377 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 8378 Operands[0], 8379 getValueType(ExtVT))); 8380 } 8381 } 8382 } 8383 8384 for (; i < ResNE; ++i) 8385 Scalars.push_back(getUNDEF(EltVT)); 8386 8387 EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE); 8388 return getBuildVector(VecVT, dl, Scalars); 8389 } 8390 8391 bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD, 8392 LoadSDNode *Base, 8393 unsigned Bytes, 8394 int Dist) const { 8395 if (LD->isVolatile() || Base->isVolatile()) 8396 return false; 8397 if (LD->isIndexed() || Base->isIndexed()) 8398 return false; 8399 if (LD->getChain() != Base->getChain()) 8400 return false; 8401 EVT VT = LD->getValueType(0); 8402 if (VT.getSizeInBits() / 8 != Bytes) 8403 return false; 8404 8405 auto BaseLocDecomp = BaseIndexOffset::match(Base, *this); 8406 auto LocDecomp = BaseIndexOffset::match(LD, *this); 8407 8408 int64_t Offset = 0; 8409 if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset)) 8410 return (Dist * Bytes == Offset); 8411 return false; 8412 } 8413 8414 /// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if 8415 /// it cannot be inferred. 8416 unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { 8417 // If this is a GlobalAddress + cst, return the alignment. 8418 const GlobalValue *GV; 8419 int64_t GVOffset = 0; 8420 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { 8421 unsigned IdxWidth = getDataLayout().getIndexTypeSizeInBits(GV->getType()); 8422 KnownBits Known(IdxWidth); 8423 llvm::computeKnownBits(GV, Known, getDataLayout()); 8424 unsigned AlignBits = Known.countMinTrailingZeros(); 8425 unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0; 8426 if (Align) 8427 return MinAlign(Align, GVOffset); 8428 } 8429 8430 // If this is a direct reference to a stack slot, use information about the 8431 // stack slot's alignment. 8432 int FrameIdx = 1 << 31; 8433 int64_t FrameOffset = 0; 8434 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) { 8435 FrameIdx = FI->getIndex(); 8436 } else if (isBaseWithConstantOffset(Ptr) && 8437 isa<FrameIndexSDNode>(Ptr.getOperand(0))) { 8438 // Handle FI+Cst 8439 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 8440 FrameOffset = Ptr.getConstantOperandVal(1); 8441 } 8442 8443 if (FrameIdx != (1 << 31)) { 8444 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo(); 8445 unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx), 8446 FrameOffset); 8447 return FIInfoAlign; 8448 } 8449 8450 return 0; 8451 } 8452 8453 /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type 8454 /// which is split (or expanded) into two not necessarily identical pieces. 8455 std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const { 8456 // Currently all types are split in half. 8457 EVT LoVT, HiVT; 8458 if (!VT.isVector()) 8459 LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT); 8460 else 8461 LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext()); 8462 8463 return std::make_pair(LoVT, HiVT); 8464 } 8465 8466 /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the 8467 /// low/high part. 8468 std::pair<SDValue, SDValue> 8469 SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, 8470 const EVT &HiVT) { 8471 assert(LoVT.getVectorNumElements() + HiVT.getVectorNumElements() <= 8472 N.getValueType().getVectorNumElements() && 8473 "More vector elements requested than available!"); 8474 SDValue Lo, Hi; 8475 Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, 8476 getConstant(0, DL, TLI->getVectorIdxTy(getDataLayout()))); 8477 Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N, 8478 getConstant(LoVT.getVectorNumElements(), DL, 8479 TLI->getVectorIdxTy(getDataLayout()))); 8480 return std::make_pair(Lo, Hi); 8481 } 8482 8483 void SelectionDAG::ExtractVectorElements(SDValue Op, 8484 SmallVectorImpl<SDValue> &Args, 8485 unsigned Start, unsigned Count) { 8486 EVT VT = Op.getValueType(); 8487 if (Count == 0) 8488 Count = VT.getVectorNumElements(); 8489 8490 EVT EltVT = VT.getVectorElementType(); 8491 EVT IdxTy = TLI->getVectorIdxTy(getDataLayout()); 8492 SDLoc SL(Op); 8493 for (unsigned i = Start, e = Start + Count; i != e; ++i) { 8494 Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, 8495 Op, getConstant(i, SL, IdxTy))); 8496 } 8497 } 8498 8499 // getAddressSpace - Return the address space this GlobalAddress belongs to. 8500 unsigned GlobalAddressSDNode::getAddressSpace() const { 8501 return getGlobal()->getType()->getAddressSpace(); 8502 } 8503 8504 Type *ConstantPoolSDNode::getType() const { 8505 if (isMachineConstantPoolEntry()) 8506 return Val.MachineCPVal->getType(); 8507 return Val.ConstVal->getType(); 8508 } 8509 8510 bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef, 8511 unsigned &SplatBitSize, 8512 bool &HasAnyUndefs, 8513 unsigned MinSplatBits, 8514 bool IsBigEndian) const { 8515 EVT VT = getValueType(0); 8516 assert(VT.isVector() && "Expected a vector type"); 8517 unsigned VecWidth = VT.getSizeInBits(); 8518 if (MinSplatBits > VecWidth) 8519 return false; 8520 8521 // FIXME: The widths are based on this node's type, but build vectors can 8522 // truncate their operands. 8523 SplatValue = APInt(VecWidth, 0); 8524 SplatUndef = APInt(VecWidth, 0); 8525 8526 // Get the bits. Bits with undefined values (when the corresponding element 8527 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared 8528 // in SplatValue. If any of the values are not constant, give up and return 8529 // false. 8530 unsigned int NumOps = getNumOperands(); 8531 assert(NumOps > 0 && "isConstantSplat has 0-size build vector"); 8532 unsigned EltWidth = VT.getScalarSizeInBits(); 8533 8534 for (unsigned j = 0; j < NumOps; ++j) { 8535 unsigned i = IsBigEndian ? NumOps - 1 - j : j; 8536 SDValue OpVal = getOperand(i); 8537 unsigned BitPos = j * EltWidth; 8538 8539 if (OpVal.isUndef()) 8540 SplatUndef.setBits(BitPos, BitPos + EltWidth); 8541 else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal)) 8542 SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos); 8543 else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal)) 8544 SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos); 8545 else 8546 return false; 8547 } 8548 8549 // The build_vector is all constants or undefs. Find the smallest element 8550 // size that splats the vector. 8551 HasAnyUndefs = (SplatUndef != 0); 8552 8553 // FIXME: This does not work for vectors with elements less than 8 bits. 8554 while (VecWidth > 8) { 8555 unsigned HalfSize = VecWidth / 2; 8556 APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize); 8557 APInt LowValue = SplatValue.trunc(HalfSize); 8558 APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize); 8559 APInt LowUndef = SplatUndef.trunc(HalfSize); 8560 8561 // If the two halves do not match (ignoring undef bits), stop here. 8562 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || 8563 MinSplatBits > HalfSize) 8564 break; 8565 8566 SplatValue = HighValue | LowValue; 8567 SplatUndef = HighUndef & LowUndef; 8568 8569 VecWidth = HalfSize; 8570 } 8571 8572 SplatBitSize = VecWidth; 8573 return true; 8574 } 8575 8576 SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const { 8577 if (UndefElements) { 8578 UndefElements->clear(); 8579 UndefElements->resize(getNumOperands()); 8580 } 8581 SDValue Splatted; 8582 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) { 8583 SDValue Op = getOperand(i); 8584 if (Op.isUndef()) { 8585 if (UndefElements) 8586 (*UndefElements)[i] = true; 8587 } else if (!Splatted) { 8588 Splatted = Op; 8589 } else if (Splatted != Op) { 8590 return SDValue(); 8591 } 8592 } 8593 8594 if (!Splatted) { 8595 assert(getOperand(0).isUndef() && 8596 "Can only have a splat without a constant for all undefs."); 8597 return getOperand(0); 8598 } 8599 8600 return Splatted; 8601 } 8602 8603 ConstantSDNode * 8604 BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const { 8605 return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements)); 8606 } 8607 8608 ConstantFPSDNode * 8609 BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const { 8610 return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements)); 8611 } 8612 8613 int32_t 8614 BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, 8615 uint32_t BitWidth) const { 8616 if (ConstantFPSDNode *CN = 8617 dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) { 8618 bool IsExact; 8619 APSInt IntVal(BitWidth); 8620 const APFloat &APF = CN->getValueAPF(); 8621 if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) != 8622 APFloat::opOK || 8623 !IsExact) 8624 return -1; 8625 8626 return IntVal.exactLogBase2(); 8627 } 8628 return -1; 8629 } 8630 8631 bool BuildVectorSDNode::isConstant() const { 8632 for (const SDValue &Op : op_values()) { 8633 unsigned Opc = Op.getOpcode(); 8634 if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP) 8635 return false; 8636 } 8637 return true; 8638 } 8639 8640 bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { 8641 // Find the first non-undef value in the shuffle mask. 8642 unsigned i, e; 8643 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) 8644 /* search */; 8645 8646 assert(i != e && "VECTOR_SHUFFLE node with all undef indices!"); 8647 8648 // Make sure all remaining elements are either undef or the same as the first 8649 // non-undef value. 8650 for (int Idx = Mask[i]; i != e; ++i) 8651 if (Mask[i] >= 0 && Mask[i] != Idx) 8652 return false; 8653 return true; 8654 } 8655 8656 // Returns the SDNode if it is a constant integer BuildVector 8657 // or constant integer. 8658 SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) { 8659 if (isa<ConstantSDNode>(N)) 8660 return N.getNode(); 8661 if (ISD::isBuildVectorOfConstantSDNodes(N.getNode())) 8662 return N.getNode(); 8663 // Treat a GlobalAddress supporting constant offset folding as a 8664 // constant integer. 8665 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N)) 8666 if (GA->getOpcode() == ISD::GlobalAddress && 8667 TLI->isOffsetFoldingLegal(GA)) 8668 return GA; 8669 return nullptr; 8670 } 8671 8672 SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) { 8673 if (isa<ConstantFPSDNode>(N)) 8674 return N.getNode(); 8675 8676 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode())) 8677 return N.getNode(); 8678 8679 return nullptr; 8680 } 8681 8682 void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) { 8683 assert(!Node->OperandList && "Node already has operands"); 8684 SDUse *Ops = OperandRecycler.allocate( 8685 ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator); 8686 8687 bool IsDivergent = false; 8688 for (unsigned I = 0; I != Vals.size(); ++I) { 8689 Ops[I].setUser(Node); 8690 Ops[I].setInitial(Vals[I]); 8691 if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence. 8692 IsDivergent = IsDivergent || Ops[I].getNode()->isDivergent(); 8693 } 8694 Node->NumOperands = Vals.size(); 8695 Node->OperandList = Ops; 8696 IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, DA); 8697 if (!TLI->isSDNodeAlwaysUniform(Node)) 8698 Node->SDNodeBits.IsDivergent = IsDivergent; 8699 checkForCycles(Node); 8700 } 8701 8702 #ifndef NDEBUG 8703 static void checkForCyclesHelper(const SDNode *N, 8704 SmallPtrSetImpl<const SDNode*> &Visited, 8705 SmallPtrSetImpl<const SDNode*> &Checked, 8706 const llvm::SelectionDAG *DAG) { 8707 // If this node has already been checked, don't check it again. 8708 if (Checked.count(N)) 8709 return; 8710 8711 // If a node has already been visited on this depth-first walk, reject it as 8712 // a cycle. 8713 if (!Visited.insert(N).second) { 8714 errs() << "Detected cycle in SelectionDAG\n"; 8715 dbgs() << "Offending node:\n"; 8716 N->dumprFull(DAG); dbgs() << "\n"; 8717 abort(); 8718 } 8719 8720 for (const SDValue &Op : N->op_values()) 8721 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG); 8722 8723 Checked.insert(N); 8724 Visited.erase(N); 8725 } 8726 #endif 8727 8728 void llvm::checkForCycles(const llvm::SDNode *N, 8729 const llvm::SelectionDAG *DAG, 8730 bool force) { 8731 #ifndef NDEBUG 8732 bool check = force; 8733 #ifdef EXPENSIVE_CHECKS 8734 check = true; 8735 #endif // EXPENSIVE_CHECKS 8736 if (check) { 8737 assert(N && "Checking nonexistent SDNode"); 8738 SmallPtrSet<const SDNode*, 32> visited; 8739 SmallPtrSet<const SDNode*, 32> checked; 8740 checkForCyclesHelper(N, visited, checked, DAG); 8741 } 8742 #endif // !NDEBUG 8743 } 8744 8745 void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) { 8746 checkForCycles(DAG->getRoot().getNode(), DAG, force); 8747 } 8748