1 //===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file contains the implementation of the scalar evolution analysis 11 // engine, which is used primarily to analyze expressions involving induction 12 // variables in loops. 13 // 14 // There are several aspects to this library. First is the representation of 15 // scalar expressions, which are represented as subclasses of the SCEV class. 16 // These classes are used to represent certain types of subexpressions that we 17 // can handle. We only create one SCEV of a particular shape, so 18 // pointer-comparisons for equality are legal. 19 // 20 // One important aspect of the SCEV objects is that they are never cyclic, even 21 // if there is a cycle in the dataflow for an expression (ie, a PHI node). If 22 // the PHI node is one of the idioms that we can represent (e.g., a polynomial 23 // recurrence) then we represent it directly as a recurrence node, otherwise we 24 // represent it as a SCEVUnknown node. 25 // 26 // In addition to being able to represent expressions of various types, we also 27 // have folders that are used to build the *canonical* representation for a 28 // particular expression. These folders are capable of using a variety of 29 // rewrite rules to simplify the expressions. 30 // 31 // Once the folders are defined, we can implement the more interesting 32 // higher-level code, such as the code that recognizes PHI nodes of various 33 // types, computes the execution count of a loop, etc. 34 // 35 // TODO: We should use these routines and value representations to implement 36 // dependence analysis! 37 // 38 //===----------------------------------------------------------------------===// 39 // 40 // There are several good references for the techniques used in this analysis. 41 // 42 // Chains of recurrences -- a method to expedite the evaluation 43 // of closed-form functions 44 // Olaf Bachmann, Paul S. Wang, Eugene V. Zima 45 // 46 // On computational properties of chains of recurrences 47 // Eugene V. Zima 48 // 49 // Symbolic Evaluation of Chains of Recurrences for Loop Optimization 50 // Robert A. van Engelen 51 // 52 // Efficient Symbolic Analysis for Optimizing Compilers 53 // Robert A. van Engelen 54 // 55 // Using the chains of recurrences algebra for data dependence testing and 56 // induction variable substitution 57 // MS Thesis, Johnie Birch 58 // 59 //===----------------------------------------------------------------------===// 60 61 #include "llvm/Analysis/ScalarEvolution.h" 62 #include "llvm/ADT/Optional.h" 63 #include "llvm/ADT/STLExtras.h" 64 #include "llvm/ADT/SmallPtrSet.h" 65 #include "llvm/ADT/Statistic.h" 66 #include "llvm/Analysis/AssumptionCache.h" 67 #include "llvm/Analysis/ConstantFolding.h" 68 #include "llvm/Analysis/InstructionSimplify.h" 69 #include "llvm/Analysis/LoopInfo.h" 70 #include "llvm/Analysis/ScalarEvolutionExpressions.h" 71 #include "llvm/Analysis/TargetLibraryInfo.h" 72 #include "llvm/Analysis/ValueTracking.h" 73 #include "llvm/IR/ConstantRange.h" 74 #include "llvm/IR/Constants.h" 75 #include "llvm/IR/DataLayout.h" 76 #include "llvm/IR/DerivedTypes.h" 77 #include "llvm/IR/Dominators.h" 78 #include "llvm/IR/GetElementPtrTypeIterator.h" 79 #include "llvm/IR/GlobalAlias.h" 80 #include "llvm/IR/GlobalVariable.h" 81 #include "llvm/IR/InstIterator.h" 82 #include "llvm/IR/Instructions.h" 83 #include "llvm/IR/LLVMContext.h" 84 #include "llvm/IR/Metadata.h" 85 #include "llvm/IR/Operator.h" 86 #include "llvm/IR/PatternMatch.h" 87 #include "llvm/Support/CommandLine.h" 88 #include "llvm/Support/Debug.h" 89 #include "llvm/Support/ErrorHandling.h" 90 #include "llvm/Support/MathExtras.h" 91 #include "llvm/Support/raw_ostream.h" 92 #include "llvm/Support/SaveAndRestore.h" 93 #include <algorithm> 94 using namespace llvm; 95 96 #define DEBUG_TYPE "scalar-evolution" 97 98 STATISTIC(NumArrayLenItCounts, 99 "Number of trip counts computed with array length"); 100 STATISTIC(NumTripCountsComputed, 101 "Number of loops with predictable loop counts"); 102 STATISTIC(NumTripCountsNotComputed, 103 "Number of loops without predictable loop counts"); 104 STATISTIC(NumBruteForceTripCountsComputed, 105 "Number of loops with trip counts computed by force"); 106 107 static cl::opt<unsigned> 108 MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden, 109 cl::desc("Maximum number of iterations SCEV will " 110 "symbolically execute a constant " 111 "derived loop"), 112 cl::init(100)); 113 114 // FIXME: Enable this with EXPENSIVE_CHECKS when the test suite is clean. 115 static cl::opt<bool> 116 VerifySCEV("verify-scev", 117 cl::desc("Verify ScalarEvolution's backedge taken counts (slow)")); 118 static cl::opt<bool> 119 VerifySCEVMap("verify-scev-maps", 120 cl::desc("Verify no dangling value in ScalarEvolution's " 121 "ExprValueMap (slow)")); 122 123 //===----------------------------------------------------------------------===// 124 // SCEV class definitions 125 //===----------------------------------------------------------------------===// 126 127 //===----------------------------------------------------------------------===// 128 // Implementation of the SCEV class. 129 // 130 131 LLVM_DUMP_METHOD 132 void SCEV::dump() const { 133 print(dbgs()); 134 dbgs() << '\n'; 135 } 136 137 void SCEV::print(raw_ostream &OS) const { 138 switch (static_cast<SCEVTypes>(getSCEVType())) { 139 case scConstant: 140 cast<SCEVConstant>(this)->getValue()->printAsOperand(OS, false); 141 return; 142 case scTruncate: { 143 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(this); 144 const SCEV *Op = Trunc->getOperand(); 145 OS << "(trunc " << *Op->getType() << " " << *Op << " to " 146 << *Trunc->getType() << ")"; 147 return; 148 } 149 case scZeroExtend: { 150 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(this); 151 const SCEV *Op = ZExt->getOperand(); 152 OS << "(zext " << *Op->getType() << " " << *Op << " to " 153 << *ZExt->getType() << ")"; 154 return; 155 } 156 case scSignExtend: { 157 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(this); 158 const SCEV *Op = SExt->getOperand(); 159 OS << "(sext " << *Op->getType() << " " << *Op << " to " 160 << *SExt->getType() << ")"; 161 return; 162 } 163 case scAddRecExpr: { 164 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(this); 165 OS << "{" << *AR->getOperand(0); 166 for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i) 167 OS << ",+," << *AR->getOperand(i); 168 OS << "}<"; 169 if (AR->hasNoUnsignedWrap()) 170 OS << "nuw><"; 171 if (AR->hasNoSignedWrap()) 172 OS << "nsw><"; 173 if (AR->hasNoSelfWrap() && 174 !AR->getNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW))) 175 OS << "nw><"; 176 AR->getLoop()->getHeader()->printAsOperand(OS, /*PrintType=*/false); 177 OS << ">"; 178 return; 179 } 180 case scAddExpr: 181 case scMulExpr: 182 case scUMaxExpr: 183 case scSMaxExpr: { 184 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(this); 185 const char *OpStr = nullptr; 186 switch (NAry->getSCEVType()) { 187 case scAddExpr: OpStr = " + "; break; 188 case scMulExpr: OpStr = " * "; break; 189 case scUMaxExpr: OpStr = " umax "; break; 190 case scSMaxExpr: OpStr = " smax "; break; 191 } 192 OS << "("; 193 for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); 194 I != E; ++I) { 195 OS << **I; 196 if (std::next(I) != E) 197 OS << OpStr; 198 } 199 OS << ")"; 200 switch (NAry->getSCEVType()) { 201 case scAddExpr: 202 case scMulExpr: 203 if (NAry->hasNoUnsignedWrap()) 204 OS << "<nuw>"; 205 if (NAry->hasNoSignedWrap()) 206 OS << "<nsw>"; 207 } 208 return; 209 } 210 case scUDivExpr: { 211 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(this); 212 OS << "(" << *UDiv->getLHS() << " /u " << *UDiv->getRHS() << ")"; 213 return; 214 } 215 case scUnknown: { 216 const SCEVUnknown *U = cast<SCEVUnknown>(this); 217 Type *AllocTy; 218 if (U->isSizeOf(AllocTy)) { 219 OS << "sizeof(" << *AllocTy << ")"; 220 return; 221 } 222 if (U->isAlignOf(AllocTy)) { 223 OS << "alignof(" << *AllocTy << ")"; 224 return; 225 } 226 227 Type *CTy; 228 Constant *FieldNo; 229 if (U->isOffsetOf(CTy, FieldNo)) { 230 OS << "offsetof(" << *CTy << ", "; 231 FieldNo->printAsOperand(OS, false); 232 OS << ")"; 233 return; 234 } 235 236 // Otherwise just print it normally. 237 U->getValue()->printAsOperand(OS, false); 238 return; 239 } 240 case scCouldNotCompute: 241 OS << "***COULDNOTCOMPUTE***"; 242 return; 243 } 244 llvm_unreachable("Unknown SCEV kind!"); 245 } 246 247 Type *SCEV::getType() const { 248 switch (static_cast<SCEVTypes>(getSCEVType())) { 249 case scConstant: 250 return cast<SCEVConstant>(this)->getType(); 251 case scTruncate: 252 case scZeroExtend: 253 case scSignExtend: 254 return cast<SCEVCastExpr>(this)->getType(); 255 case scAddRecExpr: 256 case scMulExpr: 257 case scUMaxExpr: 258 case scSMaxExpr: 259 return cast<SCEVNAryExpr>(this)->getType(); 260 case scAddExpr: 261 return cast<SCEVAddExpr>(this)->getType(); 262 case scUDivExpr: 263 return cast<SCEVUDivExpr>(this)->getType(); 264 case scUnknown: 265 return cast<SCEVUnknown>(this)->getType(); 266 case scCouldNotCompute: 267 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 268 } 269 llvm_unreachable("Unknown SCEV kind!"); 270 } 271 272 bool SCEV::isZero() const { 273 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this)) 274 return SC->getValue()->isZero(); 275 return false; 276 } 277 278 bool SCEV::isOne() const { 279 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this)) 280 return SC->getValue()->isOne(); 281 return false; 282 } 283 284 bool SCEV::isAllOnesValue() const { 285 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this)) 286 return SC->getValue()->isAllOnesValue(); 287 return false; 288 } 289 290 bool SCEV::isNonConstantNegative() const { 291 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(this); 292 if (!Mul) return false; 293 294 // If there is a constant factor, it will be first. 295 const SCEVConstant *SC = dyn_cast<SCEVConstant>(Mul->getOperand(0)); 296 if (!SC) return false; 297 298 // Return true if the value is negative, this matches things like (-42 * V). 299 return SC->getAPInt().isNegative(); 300 } 301 302 SCEVCouldNotCompute::SCEVCouldNotCompute() : 303 SCEV(FoldingSetNodeIDRef(), scCouldNotCompute) {} 304 305 bool SCEVCouldNotCompute::classof(const SCEV *S) { 306 return S->getSCEVType() == scCouldNotCompute; 307 } 308 309 const SCEV *ScalarEvolution::getConstant(ConstantInt *V) { 310 FoldingSetNodeID ID; 311 ID.AddInteger(scConstant); 312 ID.AddPointer(V); 313 void *IP = nullptr; 314 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 315 SCEV *S = new (SCEVAllocator) SCEVConstant(ID.Intern(SCEVAllocator), V); 316 UniqueSCEVs.InsertNode(S, IP); 317 return S; 318 } 319 320 const SCEV *ScalarEvolution::getConstant(const APInt &Val) { 321 return getConstant(ConstantInt::get(getContext(), Val)); 322 } 323 324 const SCEV * 325 ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) { 326 IntegerType *ITy = cast<IntegerType>(getEffectiveSCEVType(Ty)); 327 return getConstant(ConstantInt::get(ITy, V, isSigned)); 328 } 329 330 SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, 331 unsigned SCEVTy, const SCEV *op, Type *ty) 332 : SCEV(ID, SCEVTy), Op(op), Ty(ty) {} 333 334 SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, 335 const SCEV *op, Type *ty) 336 : SCEVCastExpr(ID, scTruncate, op, ty) { 337 assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) && 338 (Ty->isIntegerTy() || Ty->isPointerTy()) && 339 "Cannot truncate non-integer value!"); 340 } 341 342 SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, 343 const SCEV *op, Type *ty) 344 : SCEVCastExpr(ID, scZeroExtend, op, ty) { 345 assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) && 346 (Ty->isIntegerTy() || Ty->isPointerTy()) && 347 "Cannot zero extend non-integer value!"); 348 } 349 350 SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, 351 const SCEV *op, Type *ty) 352 : SCEVCastExpr(ID, scSignExtend, op, ty) { 353 assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) && 354 (Ty->isIntegerTy() || Ty->isPointerTy()) && 355 "Cannot sign extend non-integer value!"); 356 } 357 358 void SCEVUnknown::deleted() { 359 // Clear this SCEVUnknown from various maps. 360 SE->forgetMemoizedResults(this); 361 362 // Remove this SCEVUnknown from the uniquing map. 363 SE->UniqueSCEVs.RemoveNode(this); 364 365 // Release the value. 366 setValPtr(nullptr); 367 } 368 369 void SCEVUnknown::allUsesReplacedWith(Value *New) { 370 // Clear this SCEVUnknown from various maps. 371 SE->forgetMemoizedResults(this); 372 373 // Remove this SCEVUnknown from the uniquing map. 374 SE->UniqueSCEVs.RemoveNode(this); 375 376 // Update this SCEVUnknown to point to the new value. This is needed 377 // because there may still be outstanding SCEVs which still point to 378 // this SCEVUnknown. 379 setValPtr(New); 380 } 381 382 bool SCEVUnknown::isSizeOf(Type *&AllocTy) const { 383 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue())) 384 if (VCE->getOpcode() == Instruction::PtrToInt) 385 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0))) 386 if (CE->getOpcode() == Instruction::GetElementPtr && 387 CE->getOperand(0)->isNullValue() && 388 CE->getNumOperands() == 2) 389 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(1))) 390 if (CI->isOne()) { 391 AllocTy = cast<PointerType>(CE->getOperand(0)->getType()) 392 ->getElementType(); 393 return true; 394 } 395 396 return false; 397 } 398 399 bool SCEVUnknown::isAlignOf(Type *&AllocTy) const { 400 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue())) 401 if (VCE->getOpcode() == Instruction::PtrToInt) 402 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0))) 403 if (CE->getOpcode() == Instruction::GetElementPtr && 404 CE->getOperand(0)->isNullValue()) { 405 Type *Ty = 406 cast<PointerType>(CE->getOperand(0)->getType())->getElementType(); 407 if (StructType *STy = dyn_cast<StructType>(Ty)) 408 if (!STy->isPacked() && 409 CE->getNumOperands() == 3 && 410 CE->getOperand(1)->isNullValue()) { 411 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(2))) 412 if (CI->isOne() && 413 STy->getNumElements() == 2 && 414 STy->getElementType(0)->isIntegerTy(1)) { 415 AllocTy = STy->getElementType(1); 416 return true; 417 } 418 } 419 } 420 421 return false; 422 } 423 424 bool SCEVUnknown::isOffsetOf(Type *&CTy, Constant *&FieldNo) const { 425 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue())) 426 if (VCE->getOpcode() == Instruction::PtrToInt) 427 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0))) 428 if (CE->getOpcode() == Instruction::GetElementPtr && 429 CE->getNumOperands() == 3 && 430 CE->getOperand(0)->isNullValue() && 431 CE->getOperand(1)->isNullValue()) { 432 Type *Ty = 433 cast<PointerType>(CE->getOperand(0)->getType())->getElementType(); 434 // Ignore vector types here so that ScalarEvolutionExpander doesn't 435 // emit getelementptrs that index into vectors. 436 if (Ty->isStructTy() || Ty->isArrayTy()) { 437 CTy = Ty; 438 FieldNo = CE->getOperand(2); 439 return true; 440 } 441 } 442 443 return false; 444 } 445 446 //===----------------------------------------------------------------------===// 447 // SCEV Utilities 448 //===----------------------------------------------------------------------===// 449 450 namespace { 451 /// SCEVComplexityCompare - Return true if the complexity of the LHS is less 452 /// than the complexity of the RHS. This comparator is used to canonicalize 453 /// expressions. 454 class SCEVComplexityCompare { 455 const LoopInfo *const LI; 456 public: 457 explicit SCEVComplexityCompare(const LoopInfo *li) : LI(li) {} 458 459 // Return true or false if LHS is less than, or at least RHS, respectively. 460 bool operator()(const SCEV *LHS, const SCEV *RHS) const { 461 return compare(LHS, RHS) < 0; 462 } 463 464 // Return negative, zero, or positive, if LHS is less than, equal to, or 465 // greater than RHS, respectively. A three-way result allows recursive 466 // comparisons to be more efficient. 467 int compare(const SCEV *LHS, const SCEV *RHS) const { 468 // Fast-path: SCEVs are uniqued so we can do a quick equality check. 469 if (LHS == RHS) 470 return 0; 471 472 // Primarily, sort the SCEVs by their getSCEVType(). 473 unsigned LType = LHS->getSCEVType(), RType = RHS->getSCEVType(); 474 if (LType != RType) 475 return (int)LType - (int)RType; 476 477 // Aside from the getSCEVType() ordering, the particular ordering 478 // isn't very important except that it's beneficial to be consistent, 479 // so that (a + b) and (b + a) don't end up as different expressions. 480 switch (static_cast<SCEVTypes>(LType)) { 481 case scUnknown: { 482 const SCEVUnknown *LU = cast<SCEVUnknown>(LHS); 483 const SCEVUnknown *RU = cast<SCEVUnknown>(RHS); 484 485 // Sort SCEVUnknown values with some loose heuristics. TODO: This is 486 // not as complete as it could be. 487 const Value *LV = LU->getValue(), *RV = RU->getValue(); 488 489 // Order pointer values after integer values. This helps SCEVExpander 490 // form GEPs. 491 bool LIsPointer = LV->getType()->isPointerTy(), 492 RIsPointer = RV->getType()->isPointerTy(); 493 if (LIsPointer != RIsPointer) 494 return (int)LIsPointer - (int)RIsPointer; 495 496 // Compare getValueID values. 497 unsigned LID = LV->getValueID(), 498 RID = RV->getValueID(); 499 if (LID != RID) 500 return (int)LID - (int)RID; 501 502 // Sort arguments by their position. 503 if (const Argument *LA = dyn_cast<Argument>(LV)) { 504 const Argument *RA = cast<Argument>(RV); 505 unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo(); 506 return (int)LArgNo - (int)RArgNo; 507 } 508 509 // For instructions, compare their loop depth, and their operand 510 // count. This is pretty loose. 511 if (const Instruction *LInst = dyn_cast<Instruction>(LV)) { 512 const Instruction *RInst = cast<Instruction>(RV); 513 514 // Compare loop depths. 515 const BasicBlock *LParent = LInst->getParent(), 516 *RParent = RInst->getParent(); 517 if (LParent != RParent) { 518 unsigned LDepth = LI->getLoopDepth(LParent), 519 RDepth = LI->getLoopDepth(RParent); 520 if (LDepth != RDepth) 521 return (int)LDepth - (int)RDepth; 522 } 523 524 // Compare the number of operands. 525 unsigned LNumOps = LInst->getNumOperands(), 526 RNumOps = RInst->getNumOperands(); 527 return (int)LNumOps - (int)RNumOps; 528 } 529 530 return 0; 531 } 532 533 case scConstant: { 534 const SCEVConstant *LC = cast<SCEVConstant>(LHS); 535 const SCEVConstant *RC = cast<SCEVConstant>(RHS); 536 537 // Compare constant values. 538 const APInt &LA = LC->getAPInt(); 539 const APInt &RA = RC->getAPInt(); 540 unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth(); 541 if (LBitWidth != RBitWidth) 542 return (int)LBitWidth - (int)RBitWidth; 543 return LA.ult(RA) ? -1 : 1; 544 } 545 546 case scAddRecExpr: { 547 const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(LHS); 548 const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(RHS); 549 550 // Compare addrec loop depths. 551 const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop(); 552 if (LLoop != RLoop) { 553 unsigned LDepth = LLoop->getLoopDepth(), 554 RDepth = RLoop->getLoopDepth(); 555 if (LDepth != RDepth) 556 return (int)LDepth - (int)RDepth; 557 } 558 559 // Addrec complexity grows with operand count. 560 unsigned LNumOps = LA->getNumOperands(), RNumOps = RA->getNumOperands(); 561 if (LNumOps != RNumOps) 562 return (int)LNumOps - (int)RNumOps; 563 564 // Lexicographically compare. 565 for (unsigned i = 0; i != LNumOps; ++i) { 566 long X = compare(LA->getOperand(i), RA->getOperand(i)); 567 if (X != 0) 568 return X; 569 } 570 571 return 0; 572 } 573 574 case scAddExpr: 575 case scMulExpr: 576 case scSMaxExpr: 577 case scUMaxExpr: { 578 const SCEVNAryExpr *LC = cast<SCEVNAryExpr>(LHS); 579 const SCEVNAryExpr *RC = cast<SCEVNAryExpr>(RHS); 580 581 // Lexicographically compare n-ary expressions. 582 unsigned LNumOps = LC->getNumOperands(), RNumOps = RC->getNumOperands(); 583 if (LNumOps != RNumOps) 584 return (int)LNumOps - (int)RNumOps; 585 586 for (unsigned i = 0; i != LNumOps; ++i) { 587 if (i >= RNumOps) 588 return 1; 589 long X = compare(LC->getOperand(i), RC->getOperand(i)); 590 if (X != 0) 591 return X; 592 } 593 return (int)LNumOps - (int)RNumOps; 594 } 595 596 case scUDivExpr: { 597 const SCEVUDivExpr *LC = cast<SCEVUDivExpr>(LHS); 598 const SCEVUDivExpr *RC = cast<SCEVUDivExpr>(RHS); 599 600 // Lexicographically compare udiv expressions. 601 long X = compare(LC->getLHS(), RC->getLHS()); 602 if (X != 0) 603 return X; 604 return compare(LC->getRHS(), RC->getRHS()); 605 } 606 607 case scTruncate: 608 case scZeroExtend: 609 case scSignExtend: { 610 const SCEVCastExpr *LC = cast<SCEVCastExpr>(LHS); 611 const SCEVCastExpr *RC = cast<SCEVCastExpr>(RHS); 612 613 // Compare cast expressions by operand. 614 return compare(LC->getOperand(), RC->getOperand()); 615 } 616 617 case scCouldNotCompute: 618 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 619 } 620 llvm_unreachable("Unknown SCEV kind!"); 621 } 622 }; 623 } // end anonymous namespace 624 625 /// Given a list of SCEV objects, order them by their complexity, and group 626 /// objects of the same complexity together by value. When this routine is 627 /// finished, we know that any duplicates in the vector are consecutive and that 628 /// complexity is monotonically increasing. 629 /// 630 /// Note that we go take special precautions to ensure that we get deterministic 631 /// results from this routine. In other words, we don't want the results of 632 /// this to depend on where the addresses of various SCEV objects happened to 633 /// land in memory. 634 /// 635 static void GroupByComplexity(SmallVectorImpl<const SCEV *> &Ops, 636 LoopInfo *LI) { 637 if (Ops.size() < 2) return; // Noop 638 if (Ops.size() == 2) { 639 // This is the common case, which also happens to be trivially simple. 640 // Special case it. 641 const SCEV *&LHS = Ops[0], *&RHS = Ops[1]; 642 if (SCEVComplexityCompare(LI)(RHS, LHS)) 643 std::swap(LHS, RHS); 644 return; 645 } 646 647 // Do the rough sort by complexity. 648 std::stable_sort(Ops.begin(), Ops.end(), SCEVComplexityCompare(LI)); 649 650 // Now that we are sorted by complexity, group elements of the same 651 // complexity. Note that this is, at worst, N^2, but the vector is likely to 652 // be extremely short in practice. Note that we take this approach because we 653 // do not want to depend on the addresses of the objects we are grouping. 654 for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) { 655 const SCEV *S = Ops[i]; 656 unsigned Complexity = S->getSCEVType(); 657 658 // If there are any objects of the same complexity and same value as this 659 // one, group them. 660 for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) { 661 if (Ops[j] == S) { // Found a duplicate. 662 // Move it to immediately after i'th element. 663 std::swap(Ops[i+1], Ops[j]); 664 ++i; // no need to rescan it. 665 if (i == e-2) return; // Done! 666 } 667 } 668 } 669 } 670 671 // Returns the size of the SCEV S. 672 static inline int sizeOfSCEV(const SCEV *S) { 673 struct FindSCEVSize { 674 int Size; 675 FindSCEVSize() : Size(0) {} 676 677 bool follow(const SCEV *S) { 678 ++Size; 679 // Keep looking at all operands of S. 680 return true; 681 } 682 bool isDone() const { 683 return false; 684 } 685 }; 686 687 FindSCEVSize F; 688 SCEVTraversal<FindSCEVSize> ST(F); 689 ST.visitAll(S); 690 return F.Size; 691 } 692 693 namespace { 694 695 struct SCEVDivision : public SCEVVisitor<SCEVDivision, void> { 696 public: 697 // Computes the Quotient and Remainder of the division of Numerator by 698 // Denominator. 699 static void divide(ScalarEvolution &SE, const SCEV *Numerator, 700 const SCEV *Denominator, const SCEV **Quotient, 701 const SCEV **Remainder) { 702 assert(Numerator && Denominator && "Uninitialized SCEV"); 703 704 SCEVDivision D(SE, Numerator, Denominator); 705 706 // Check for the trivial case here to avoid having to check for it in the 707 // rest of the code. 708 if (Numerator == Denominator) { 709 *Quotient = D.One; 710 *Remainder = D.Zero; 711 return; 712 } 713 714 if (Numerator->isZero()) { 715 *Quotient = D.Zero; 716 *Remainder = D.Zero; 717 return; 718 } 719 720 // A simple case when N/1. The quotient is N. 721 if (Denominator->isOne()) { 722 *Quotient = Numerator; 723 *Remainder = D.Zero; 724 return; 725 } 726 727 // Split the Denominator when it is a product. 728 if (const SCEVMulExpr *T = dyn_cast<SCEVMulExpr>(Denominator)) { 729 const SCEV *Q, *R; 730 *Quotient = Numerator; 731 for (const SCEV *Op : T->operands()) { 732 divide(SE, *Quotient, Op, &Q, &R); 733 *Quotient = Q; 734 735 // Bail out when the Numerator is not divisible by one of the terms of 736 // the Denominator. 737 if (!R->isZero()) { 738 *Quotient = D.Zero; 739 *Remainder = Numerator; 740 return; 741 } 742 } 743 *Remainder = D.Zero; 744 return; 745 } 746 747 D.visit(Numerator); 748 *Quotient = D.Quotient; 749 *Remainder = D.Remainder; 750 } 751 752 // Except in the trivial case described above, we do not know how to divide 753 // Expr by Denominator for the following functions with empty implementation. 754 void visitTruncateExpr(const SCEVTruncateExpr *Numerator) {} 755 void visitZeroExtendExpr(const SCEVZeroExtendExpr *Numerator) {} 756 void visitSignExtendExpr(const SCEVSignExtendExpr *Numerator) {} 757 void visitUDivExpr(const SCEVUDivExpr *Numerator) {} 758 void visitSMaxExpr(const SCEVSMaxExpr *Numerator) {} 759 void visitUMaxExpr(const SCEVUMaxExpr *Numerator) {} 760 void visitUnknown(const SCEVUnknown *Numerator) {} 761 void visitCouldNotCompute(const SCEVCouldNotCompute *Numerator) {} 762 763 void visitConstant(const SCEVConstant *Numerator) { 764 if (const SCEVConstant *D = dyn_cast<SCEVConstant>(Denominator)) { 765 APInt NumeratorVal = Numerator->getAPInt(); 766 APInt DenominatorVal = D->getAPInt(); 767 uint32_t NumeratorBW = NumeratorVal.getBitWidth(); 768 uint32_t DenominatorBW = DenominatorVal.getBitWidth(); 769 770 if (NumeratorBW > DenominatorBW) 771 DenominatorVal = DenominatorVal.sext(NumeratorBW); 772 else if (NumeratorBW < DenominatorBW) 773 NumeratorVal = NumeratorVal.sext(DenominatorBW); 774 775 APInt QuotientVal(NumeratorVal.getBitWidth(), 0); 776 APInt RemainderVal(NumeratorVal.getBitWidth(), 0); 777 APInt::sdivrem(NumeratorVal, DenominatorVal, QuotientVal, RemainderVal); 778 Quotient = SE.getConstant(QuotientVal); 779 Remainder = SE.getConstant(RemainderVal); 780 return; 781 } 782 } 783 784 void visitAddRecExpr(const SCEVAddRecExpr *Numerator) { 785 const SCEV *StartQ, *StartR, *StepQ, *StepR; 786 if (!Numerator->isAffine()) 787 return cannotDivide(Numerator); 788 divide(SE, Numerator->getStart(), Denominator, &StartQ, &StartR); 789 divide(SE, Numerator->getStepRecurrence(SE), Denominator, &StepQ, &StepR); 790 // Bail out if the types do not match. 791 Type *Ty = Denominator->getType(); 792 if (Ty != StartQ->getType() || Ty != StartR->getType() || 793 Ty != StepQ->getType() || Ty != StepR->getType()) 794 return cannotDivide(Numerator); 795 Quotient = SE.getAddRecExpr(StartQ, StepQ, Numerator->getLoop(), 796 Numerator->getNoWrapFlags()); 797 Remainder = SE.getAddRecExpr(StartR, StepR, Numerator->getLoop(), 798 Numerator->getNoWrapFlags()); 799 } 800 801 void visitAddExpr(const SCEVAddExpr *Numerator) { 802 SmallVector<const SCEV *, 2> Qs, Rs; 803 Type *Ty = Denominator->getType(); 804 805 for (const SCEV *Op : Numerator->operands()) { 806 const SCEV *Q, *R; 807 divide(SE, Op, Denominator, &Q, &R); 808 809 // Bail out if types do not match. 810 if (Ty != Q->getType() || Ty != R->getType()) 811 return cannotDivide(Numerator); 812 813 Qs.push_back(Q); 814 Rs.push_back(R); 815 } 816 817 if (Qs.size() == 1) { 818 Quotient = Qs[0]; 819 Remainder = Rs[0]; 820 return; 821 } 822 823 Quotient = SE.getAddExpr(Qs); 824 Remainder = SE.getAddExpr(Rs); 825 } 826 827 void visitMulExpr(const SCEVMulExpr *Numerator) { 828 SmallVector<const SCEV *, 2> Qs; 829 Type *Ty = Denominator->getType(); 830 831 bool FoundDenominatorTerm = false; 832 for (const SCEV *Op : Numerator->operands()) { 833 // Bail out if types do not match. 834 if (Ty != Op->getType()) 835 return cannotDivide(Numerator); 836 837 if (FoundDenominatorTerm) { 838 Qs.push_back(Op); 839 continue; 840 } 841 842 // Check whether Denominator divides one of the product operands. 843 const SCEV *Q, *R; 844 divide(SE, Op, Denominator, &Q, &R); 845 if (!R->isZero()) { 846 Qs.push_back(Op); 847 continue; 848 } 849 850 // Bail out if types do not match. 851 if (Ty != Q->getType()) 852 return cannotDivide(Numerator); 853 854 FoundDenominatorTerm = true; 855 Qs.push_back(Q); 856 } 857 858 if (FoundDenominatorTerm) { 859 Remainder = Zero; 860 if (Qs.size() == 1) 861 Quotient = Qs[0]; 862 else 863 Quotient = SE.getMulExpr(Qs); 864 return; 865 } 866 867 if (!isa<SCEVUnknown>(Denominator)) 868 return cannotDivide(Numerator); 869 870 // The Remainder is obtained by replacing Denominator by 0 in Numerator. 871 ValueToValueMap RewriteMap; 872 RewriteMap[cast<SCEVUnknown>(Denominator)->getValue()] = 873 cast<SCEVConstant>(Zero)->getValue(); 874 Remainder = SCEVParameterRewriter::rewrite(Numerator, SE, RewriteMap, true); 875 876 if (Remainder->isZero()) { 877 // The Quotient is obtained by replacing Denominator by 1 in Numerator. 878 RewriteMap[cast<SCEVUnknown>(Denominator)->getValue()] = 879 cast<SCEVConstant>(One)->getValue(); 880 Quotient = 881 SCEVParameterRewriter::rewrite(Numerator, SE, RewriteMap, true); 882 return; 883 } 884 885 // Quotient is (Numerator - Remainder) divided by Denominator. 886 const SCEV *Q, *R; 887 const SCEV *Diff = SE.getMinusSCEV(Numerator, Remainder); 888 // This SCEV does not seem to simplify: fail the division here. 889 if (sizeOfSCEV(Diff) > sizeOfSCEV(Numerator)) 890 return cannotDivide(Numerator); 891 divide(SE, Diff, Denominator, &Q, &R); 892 if (R != Zero) 893 return cannotDivide(Numerator); 894 Quotient = Q; 895 } 896 897 private: 898 SCEVDivision(ScalarEvolution &S, const SCEV *Numerator, 899 const SCEV *Denominator) 900 : SE(S), Denominator(Denominator) { 901 Zero = SE.getZero(Denominator->getType()); 902 One = SE.getOne(Denominator->getType()); 903 904 // We generally do not know how to divide Expr by Denominator. We 905 // initialize the division to a "cannot divide" state to simplify the rest 906 // of the code. 907 cannotDivide(Numerator); 908 } 909 910 // Convenience function for giving up on the division. We set the quotient to 911 // be equal to zero and the remainder to be equal to the numerator. 912 void cannotDivide(const SCEV *Numerator) { 913 Quotient = Zero; 914 Remainder = Numerator; 915 } 916 917 ScalarEvolution &SE; 918 const SCEV *Denominator, *Quotient, *Remainder, *Zero, *One; 919 }; 920 921 } 922 923 //===----------------------------------------------------------------------===// 924 // Simple SCEV method implementations 925 //===----------------------------------------------------------------------===// 926 927 /// Compute BC(It, K). The result has width W. Assume, K > 0. 928 static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K, 929 ScalarEvolution &SE, 930 Type *ResultTy) { 931 // Handle the simplest case efficiently. 932 if (K == 1) 933 return SE.getTruncateOrZeroExtend(It, ResultTy); 934 935 // We are using the following formula for BC(It, K): 936 // 937 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K! 938 // 939 // Suppose, W is the bitwidth of the return value. We must be prepared for 940 // overflow. Hence, we must assure that the result of our computation is 941 // equal to the accurate one modulo 2^W. Unfortunately, division isn't 942 // safe in modular arithmetic. 943 // 944 // However, this code doesn't use exactly that formula; the formula it uses 945 // is something like the following, where T is the number of factors of 2 in 946 // K! (i.e. trailing zeros in the binary representation of K!), and ^ is 947 // exponentiation: 948 // 949 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T) 950 // 951 // This formula is trivially equivalent to the previous formula. However, 952 // this formula can be implemented much more efficiently. The trick is that 953 // K! / 2^T is odd, and exact division by an odd number *is* safe in modular 954 // arithmetic. To do exact division in modular arithmetic, all we have 955 // to do is multiply by the inverse. Therefore, this step can be done at 956 // width W. 957 // 958 // The next issue is how to safely do the division by 2^T. The way this 959 // is done is by doing the multiplication step at a width of at least W + T 960 // bits. This way, the bottom W+T bits of the product are accurate. Then, 961 // when we perform the division by 2^T (which is equivalent to a right shift 962 // by T), the bottom W bits are accurate. Extra bits are okay; they'll get 963 // truncated out after the division by 2^T. 964 // 965 // In comparison to just directly using the first formula, this technique 966 // is much more efficient; using the first formula requires W * K bits, 967 // but this formula less than W + K bits. Also, the first formula requires 968 // a division step, whereas this formula only requires multiplies and shifts. 969 // 970 // It doesn't matter whether the subtraction step is done in the calculation 971 // width or the input iteration count's width; if the subtraction overflows, 972 // the result must be zero anyway. We prefer here to do it in the width of 973 // the induction variable because it helps a lot for certain cases; CodeGen 974 // isn't smart enough to ignore the overflow, which leads to much less 975 // efficient code if the width of the subtraction is wider than the native 976 // register width. 977 // 978 // (It's possible to not widen at all by pulling out factors of 2 before 979 // the multiplication; for example, K=2 can be calculated as 980 // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires 981 // extra arithmetic, so it's not an obvious win, and it gets 982 // much more complicated for K > 3.) 983 984 // Protection from insane SCEVs; this bound is conservative, 985 // but it probably doesn't matter. 986 if (K > 1000) 987 return SE.getCouldNotCompute(); 988 989 unsigned W = SE.getTypeSizeInBits(ResultTy); 990 991 // Calculate K! / 2^T and T; we divide out the factors of two before 992 // multiplying for calculating K! / 2^T to avoid overflow. 993 // Other overflow doesn't matter because we only care about the bottom 994 // W bits of the result. 995 APInt OddFactorial(W, 1); 996 unsigned T = 1; 997 for (unsigned i = 3; i <= K; ++i) { 998 APInt Mult(W, i); 999 unsigned TwoFactors = Mult.countTrailingZeros(); 1000 T += TwoFactors; 1001 Mult = Mult.lshr(TwoFactors); 1002 OddFactorial *= Mult; 1003 } 1004 1005 // We need at least W + T bits for the multiplication step 1006 unsigned CalculationBits = W + T; 1007 1008 // Calculate 2^T, at width T+W. 1009 APInt DivFactor = APInt::getOneBitSet(CalculationBits, T); 1010 1011 // Calculate the multiplicative inverse of K! / 2^T; 1012 // this multiplication factor will perform the exact division by 1013 // K! / 2^T. 1014 APInt Mod = APInt::getSignedMinValue(W+1); 1015 APInt MultiplyFactor = OddFactorial.zext(W+1); 1016 MultiplyFactor = MultiplyFactor.multiplicativeInverse(Mod); 1017 MultiplyFactor = MultiplyFactor.trunc(W); 1018 1019 // Calculate the product, at width T+W 1020 IntegerType *CalculationTy = IntegerType::get(SE.getContext(), 1021 CalculationBits); 1022 const SCEV *Dividend = SE.getTruncateOrZeroExtend(It, CalculationTy); 1023 for (unsigned i = 1; i != K; ++i) { 1024 const SCEV *S = SE.getMinusSCEV(It, SE.getConstant(It->getType(), i)); 1025 Dividend = SE.getMulExpr(Dividend, 1026 SE.getTruncateOrZeroExtend(S, CalculationTy)); 1027 } 1028 1029 // Divide by 2^T 1030 const SCEV *DivResult = SE.getUDivExpr(Dividend, SE.getConstant(DivFactor)); 1031 1032 // Truncate the result, and divide by K! / 2^T. 1033 1034 return SE.getMulExpr(SE.getConstant(MultiplyFactor), 1035 SE.getTruncateOrZeroExtend(DivResult, ResultTy)); 1036 } 1037 1038 /// Return the value of this chain of recurrences at the specified iteration 1039 /// number. We can evaluate this recurrence by multiplying each element in the 1040 /// chain by the binomial coefficient corresponding to it. In other words, we 1041 /// can evaluate {A,+,B,+,C,+,D} as: 1042 /// 1043 /// A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3) 1044 /// 1045 /// where BC(It, k) stands for binomial coefficient. 1046 /// 1047 const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It, 1048 ScalarEvolution &SE) const { 1049 const SCEV *Result = getStart(); 1050 for (unsigned i = 1, e = getNumOperands(); i != e; ++i) { 1051 // The computation is correct in the face of overflow provided that the 1052 // multiplication is performed _after_ the evaluation of the binomial 1053 // coefficient. 1054 const SCEV *Coeff = BinomialCoefficient(It, i, SE, getType()); 1055 if (isa<SCEVCouldNotCompute>(Coeff)) 1056 return Coeff; 1057 1058 Result = SE.getAddExpr(Result, SE.getMulExpr(getOperand(i), Coeff)); 1059 } 1060 return Result; 1061 } 1062 1063 //===----------------------------------------------------------------------===// 1064 // SCEV Expression folder implementations 1065 //===----------------------------------------------------------------------===// 1066 1067 const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, 1068 Type *Ty) { 1069 assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) && 1070 "This is not a truncating conversion!"); 1071 assert(isSCEVable(Ty) && 1072 "This is not a conversion to a SCEVable type!"); 1073 Ty = getEffectiveSCEVType(Ty); 1074 1075 FoldingSetNodeID ID; 1076 ID.AddInteger(scTruncate); 1077 ID.AddPointer(Op); 1078 ID.AddPointer(Ty); 1079 void *IP = nullptr; 1080 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1081 1082 // Fold if the operand is constant. 1083 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 1084 return getConstant( 1085 cast<ConstantInt>(ConstantExpr::getTrunc(SC->getValue(), Ty))); 1086 1087 // trunc(trunc(x)) --> trunc(x) 1088 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) 1089 return getTruncateExpr(ST->getOperand(), Ty); 1090 1091 // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing 1092 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op)) 1093 return getTruncateOrSignExtend(SS->getOperand(), Ty); 1094 1095 // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing 1096 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op)) 1097 return getTruncateOrZeroExtend(SZ->getOperand(), Ty); 1098 1099 // trunc(x1+x2+...+xN) --> trunc(x1)+trunc(x2)+...+trunc(xN) if we can 1100 // eliminate all the truncates, or we replace other casts with truncates. 1101 if (const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Op)) { 1102 SmallVector<const SCEV *, 4> Operands; 1103 bool hasTrunc = false; 1104 for (unsigned i = 0, e = SA->getNumOperands(); i != e && !hasTrunc; ++i) { 1105 const SCEV *S = getTruncateExpr(SA->getOperand(i), Ty); 1106 if (!isa<SCEVCastExpr>(SA->getOperand(i))) 1107 hasTrunc = isa<SCEVTruncateExpr>(S); 1108 Operands.push_back(S); 1109 } 1110 if (!hasTrunc) 1111 return getAddExpr(Operands); 1112 UniqueSCEVs.FindNodeOrInsertPos(ID, IP); // Mutates IP, returns NULL. 1113 } 1114 1115 // trunc(x1*x2*...*xN) --> trunc(x1)*trunc(x2)*...*trunc(xN) if we can 1116 // eliminate all the truncates, or we replace other casts with truncates. 1117 if (const SCEVMulExpr *SM = dyn_cast<SCEVMulExpr>(Op)) { 1118 SmallVector<const SCEV *, 4> Operands; 1119 bool hasTrunc = false; 1120 for (unsigned i = 0, e = SM->getNumOperands(); i != e && !hasTrunc; ++i) { 1121 const SCEV *S = getTruncateExpr(SM->getOperand(i), Ty); 1122 if (!isa<SCEVCastExpr>(SM->getOperand(i))) 1123 hasTrunc = isa<SCEVTruncateExpr>(S); 1124 Operands.push_back(S); 1125 } 1126 if (!hasTrunc) 1127 return getMulExpr(Operands); 1128 UniqueSCEVs.FindNodeOrInsertPos(ID, IP); // Mutates IP, returns NULL. 1129 } 1130 1131 // If the input value is a chrec scev, truncate the chrec's operands. 1132 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) { 1133 SmallVector<const SCEV *, 4> Operands; 1134 for (const SCEV *Op : AddRec->operands()) 1135 Operands.push_back(getTruncateExpr(Op, Ty)); 1136 return getAddRecExpr(Operands, AddRec->getLoop(), SCEV::FlagAnyWrap); 1137 } 1138 1139 // The cast wasn't folded; create an explicit cast node. We can reuse 1140 // the existing insert position since if we get here, we won't have 1141 // made any changes which would invalidate it. 1142 SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator), 1143 Op, Ty); 1144 UniqueSCEVs.InsertNode(S, IP); 1145 return S; 1146 } 1147 1148 // Get the limit of a recurrence such that incrementing by Step cannot cause 1149 // signed overflow as long as the value of the recurrence within the 1150 // loop does not exceed this limit before incrementing. 1151 static const SCEV *getSignedOverflowLimitForStep(const SCEV *Step, 1152 ICmpInst::Predicate *Pred, 1153 ScalarEvolution *SE) { 1154 unsigned BitWidth = SE->getTypeSizeInBits(Step->getType()); 1155 if (SE->isKnownPositive(Step)) { 1156 *Pred = ICmpInst::ICMP_SLT; 1157 return SE->getConstant(APInt::getSignedMinValue(BitWidth) - 1158 SE->getSignedRange(Step).getSignedMax()); 1159 } 1160 if (SE->isKnownNegative(Step)) { 1161 *Pred = ICmpInst::ICMP_SGT; 1162 return SE->getConstant(APInt::getSignedMaxValue(BitWidth) - 1163 SE->getSignedRange(Step).getSignedMin()); 1164 } 1165 return nullptr; 1166 } 1167 1168 // Get the limit of a recurrence such that incrementing by Step cannot cause 1169 // unsigned overflow as long as the value of the recurrence within the loop does 1170 // not exceed this limit before incrementing. 1171 static const SCEV *getUnsignedOverflowLimitForStep(const SCEV *Step, 1172 ICmpInst::Predicate *Pred, 1173 ScalarEvolution *SE) { 1174 unsigned BitWidth = SE->getTypeSizeInBits(Step->getType()); 1175 *Pred = ICmpInst::ICMP_ULT; 1176 1177 return SE->getConstant(APInt::getMinValue(BitWidth) - 1178 SE->getUnsignedRange(Step).getUnsignedMax()); 1179 } 1180 1181 namespace { 1182 1183 struct ExtendOpTraitsBase { 1184 typedef const SCEV *(ScalarEvolution::*GetExtendExprTy)(const SCEV *, Type *); 1185 }; 1186 1187 // Used to make code generic over signed and unsigned overflow. 1188 template <typename ExtendOp> struct ExtendOpTraits { 1189 // Members present: 1190 // 1191 // static const SCEV::NoWrapFlags WrapType; 1192 // 1193 // static const ExtendOpTraitsBase::GetExtendExprTy GetExtendExpr; 1194 // 1195 // static const SCEV *getOverflowLimitForStep(const SCEV *Step, 1196 // ICmpInst::Predicate *Pred, 1197 // ScalarEvolution *SE); 1198 }; 1199 1200 template <> 1201 struct ExtendOpTraits<SCEVSignExtendExpr> : public ExtendOpTraitsBase { 1202 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNSW; 1203 1204 static const GetExtendExprTy GetExtendExpr; 1205 1206 static const SCEV *getOverflowLimitForStep(const SCEV *Step, 1207 ICmpInst::Predicate *Pred, 1208 ScalarEvolution *SE) { 1209 return getSignedOverflowLimitForStep(Step, Pred, SE); 1210 } 1211 }; 1212 1213 const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits< 1214 SCEVSignExtendExpr>::GetExtendExpr = &ScalarEvolution::getSignExtendExpr; 1215 1216 template <> 1217 struct ExtendOpTraits<SCEVZeroExtendExpr> : public ExtendOpTraitsBase { 1218 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNUW; 1219 1220 static const GetExtendExprTy GetExtendExpr; 1221 1222 static const SCEV *getOverflowLimitForStep(const SCEV *Step, 1223 ICmpInst::Predicate *Pred, 1224 ScalarEvolution *SE) { 1225 return getUnsignedOverflowLimitForStep(Step, Pred, SE); 1226 } 1227 }; 1228 1229 const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits< 1230 SCEVZeroExtendExpr>::GetExtendExpr = &ScalarEvolution::getZeroExtendExpr; 1231 } 1232 1233 // The recurrence AR has been shown to have no signed/unsigned wrap or something 1234 // close to it. Typically, if we can prove NSW/NUW for AR, then we can just as 1235 // easily prove NSW/NUW for its preincrement or postincrement sibling. This 1236 // allows normalizing a sign/zero extended AddRec as such: {sext/zext(Step + 1237 // Start),+,Step} => {(Step + sext/zext(Start),+,Step} As a result, the 1238 // expression "Step + sext/zext(PreIncAR)" is congruent with 1239 // "sext/zext(PostIncAR)" 1240 template <typename ExtendOpTy> 1241 static const SCEV *getPreStartForExtend(const SCEVAddRecExpr *AR, Type *Ty, 1242 ScalarEvolution *SE) { 1243 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType; 1244 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr; 1245 1246 const Loop *L = AR->getLoop(); 1247 const SCEV *Start = AR->getStart(); 1248 const SCEV *Step = AR->getStepRecurrence(*SE); 1249 1250 // Check for a simple looking step prior to loop entry. 1251 const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Start); 1252 if (!SA) 1253 return nullptr; 1254 1255 // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV 1256 // subtraction is expensive. For this purpose, perform a quick and dirty 1257 // difference, by checking for Step in the operand list. 1258 SmallVector<const SCEV *, 4> DiffOps; 1259 for (const SCEV *Op : SA->operands()) 1260 if (Op != Step) 1261 DiffOps.push_back(Op); 1262 1263 if (DiffOps.size() == SA->getNumOperands()) 1264 return nullptr; 1265 1266 // Try to prove `WrapType` (SCEV::FlagNSW or SCEV::FlagNUW) on `PreStart` + 1267 // `Step`: 1268 1269 // 1. NSW/NUW flags on the step increment. 1270 auto PreStartFlags = 1271 ScalarEvolution::maskFlags(SA->getNoWrapFlags(), SCEV::FlagNUW); 1272 const SCEV *PreStart = SE->getAddExpr(DiffOps, PreStartFlags); 1273 const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>( 1274 SE->getAddRecExpr(PreStart, Step, L, SCEV::FlagAnyWrap)); 1275 1276 // "{S,+,X} is <nsw>/<nuw>" and "the backedge is taken at least once" implies 1277 // "S+X does not sign/unsign-overflow". 1278 // 1279 1280 const SCEV *BECount = SE->getBackedgeTakenCount(L); 1281 if (PreAR && PreAR->getNoWrapFlags(WrapType) && 1282 !isa<SCEVCouldNotCompute>(BECount) && SE->isKnownPositive(BECount)) 1283 return PreStart; 1284 1285 // 2. Direct overflow check on the step operation's expression. 1286 unsigned BitWidth = SE->getTypeSizeInBits(AR->getType()); 1287 Type *WideTy = IntegerType::get(SE->getContext(), BitWidth * 2); 1288 const SCEV *OperandExtendedStart = 1289 SE->getAddExpr((SE->*GetExtendExpr)(PreStart, WideTy), 1290 (SE->*GetExtendExpr)(Step, WideTy)); 1291 if ((SE->*GetExtendExpr)(Start, WideTy) == OperandExtendedStart) { 1292 if (PreAR && AR->getNoWrapFlags(WrapType)) { 1293 // If we know `AR` == {`PreStart`+`Step`,+,`Step`} is `WrapType` (FlagNSW 1294 // or FlagNUW) and that `PreStart` + `Step` is `WrapType` too, then 1295 // `PreAR` == {`PreStart`,+,`Step`} is also `WrapType`. Cache this fact. 1296 const_cast<SCEVAddRecExpr *>(PreAR)->setNoWrapFlags(WrapType); 1297 } 1298 return PreStart; 1299 } 1300 1301 // 3. Loop precondition. 1302 ICmpInst::Predicate Pred; 1303 const SCEV *OverflowLimit = 1304 ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(Step, &Pred, SE); 1305 1306 if (OverflowLimit && 1307 SE->isLoopEntryGuardedByCond(L, Pred, PreStart, OverflowLimit)) 1308 return PreStart; 1309 1310 return nullptr; 1311 } 1312 1313 // Get the normalized zero or sign extended expression for this AddRec's Start. 1314 template <typename ExtendOpTy> 1315 static const SCEV *getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty, 1316 ScalarEvolution *SE) { 1317 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr; 1318 1319 const SCEV *PreStart = getPreStartForExtend<ExtendOpTy>(AR, Ty, SE); 1320 if (!PreStart) 1321 return (SE->*GetExtendExpr)(AR->getStart(), Ty); 1322 1323 return SE->getAddExpr((SE->*GetExtendExpr)(AR->getStepRecurrence(*SE), Ty), 1324 (SE->*GetExtendExpr)(PreStart, Ty)); 1325 } 1326 1327 // Try to prove away overflow by looking at "nearby" add recurrences. A 1328 // motivating example for this rule: if we know `{0,+,4}` is `ult` `-1` and it 1329 // does not itself wrap then we can conclude that `{1,+,4}` is `nuw`. 1330 // 1331 // Formally: 1332 // 1333 // {S,+,X} == {S-T,+,X} + T 1334 // => Ext({S,+,X}) == Ext({S-T,+,X} + T) 1335 // 1336 // If ({S-T,+,X} + T) does not overflow ... (1) 1337 // 1338 // RHS == Ext({S-T,+,X} + T) == Ext({S-T,+,X}) + Ext(T) 1339 // 1340 // If {S-T,+,X} does not overflow ... (2) 1341 // 1342 // RHS == Ext({S-T,+,X}) + Ext(T) == {Ext(S-T),+,Ext(X)} + Ext(T) 1343 // == {Ext(S-T)+Ext(T),+,Ext(X)} 1344 // 1345 // If (S-T)+T does not overflow ... (3) 1346 // 1347 // RHS == {Ext(S-T)+Ext(T),+,Ext(X)} == {Ext(S-T+T),+,Ext(X)} 1348 // == {Ext(S),+,Ext(X)} == LHS 1349 // 1350 // Thus, if (1), (2) and (3) are true for some T, then 1351 // Ext({S,+,X}) == {Ext(S),+,Ext(X)} 1352 // 1353 // (3) is implied by (1) -- "(S-T)+T does not overflow" is simply "({S-T,+,X}+T) 1354 // does not overflow" restricted to the 0th iteration. Therefore we only need 1355 // to check for (1) and (2). 1356 // 1357 // In the current context, S is `Start`, X is `Step`, Ext is `ExtendOpTy` and T 1358 // is `Delta` (defined below). 1359 // 1360 template <typename ExtendOpTy> 1361 bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start, 1362 const SCEV *Step, 1363 const Loop *L) { 1364 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType; 1365 1366 // We restrict `Start` to a constant to prevent SCEV from spending too much 1367 // time here. It is correct (but more expensive) to continue with a 1368 // non-constant `Start` and do a general SCEV subtraction to compute 1369 // `PreStart` below. 1370 // 1371 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start); 1372 if (!StartC) 1373 return false; 1374 1375 APInt StartAI = StartC->getAPInt(); 1376 1377 for (unsigned Delta : {-2, -1, 1, 2}) { 1378 const SCEV *PreStart = getConstant(StartAI - Delta); 1379 1380 FoldingSetNodeID ID; 1381 ID.AddInteger(scAddRecExpr); 1382 ID.AddPointer(PreStart); 1383 ID.AddPointer(Step); 1384 ID.AddPointer(L); 1385 void *IP = nullptr; 1386 const auto *PreAR = 1387 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 1388 1389 // Give up if we don't already have the add recurrence we need because 1390 // actually constructing an add recurrence is relatively expensive. 1391 if (PreAR && PreAR->getNoWrapFlags(WrapType)) { // proves (2) 1392 const SCEV *DeltaS = getConstant(StartC->getType(), Delta); 1393 ICmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE; 1394 const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep( 1395 DeltaS, &Pred, this); 1396 if (Limit && isKnownPredicate(Pred, PreAR, Limit)) // proves (1) 1397 return true; 1398 } 1399 } 1400 1401 return false; 1402 } 1403 1404 const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op, 1405 Type *Ty) { 1406 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && 1407 "This is not an extending conversion!"); 1408 assert(isSCEVable(Ty) && 1409 "This is not a conversion to a SCEVable type!"); 1410 Ty = getEffectiveSCEVType(Ty); 1411 1412 // Fold if the operand is constant. 1413 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 1414 return getConstant( 1415 cast<ConstantInt>(ConstantExpr::getZExt(SC->getValue(), Ty))); 1416 1417 // zext(zext(x)) --> zext(x) 1418 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op)) 1419 return getZeroExtendExpr(SZ->getOperand(), Ty); 1420 1421 // Before doing any expensive analysis, check to see if we've already 1422 // computed a SCEV for this Op and Ty. 1423 FoldingSetNodeID ID; 1424 ID.AddInteger(scZeroExtend); 1425 ID.AddPointer(Op); 1426 ID.AddPointer(Ty); 1427 void *IP = nullptr; 1428 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1429 1430 // zext(trunc(x)) --> zext(x) or x or trunc(x) 1431 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) { 1432 // It's possible the bits taken off by the truncate were all zero bits. If 1433 // so, we should be able to simplify this further. 1434 const SCEV *X = ST->getOperand(); 1435 ConstantRange CR = getUnsignedRange(X); 1436 unsigned TruncBits = getTypeSizeInBits(ST->getType()); 1437 unsigned NewBits = getTypeSizeInBits(Ty); 1438 if (CR.truncate(TruncBits).zeroExtend(NewBits).contains( 1439 CR.zextOrTrunc(NewBits))) 1440 return getTruncateOrZeroExtend(X, Ty); 1441 } 1442 1443 // If the input value is a chrec scev, and we can prove that the value 1444 // did not overflow the old, smaller, value, we can zero extend all of the 1445 // operands (often constants). This allows analysis of something like 1446 // this: for (unsigned char X = 0; X < 100; ++X) { int Y = X; } 1447 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) 1448 if (AR->isAffine()) { 1449 const SCEV *Start = AR->getStart(); 1450 const SCEV *Step = AR->getStepRecurrence(*this); 1451 unsigned BitWidth = getTypeSizeInBits(AR->getType()); 1452 const Loop *L = AR->getLoop(); 1453 1454 if (!AR->hasNoUnsignedWrap()) { 1455 auto NewFlags = proveNoWrapViaConstantRanges(AR); 1456 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(NewFlags); 1457 } 1458 1459 // If we have special knowledge that this addrec won't overflow, 1460 // we don't need to do any further analysis. 1461 if (AR->hasNoUnsignedWrap()) 1462 return getAddRecExpr( 1463 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this), 1464 getZeroExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1465 1466 // Check whether the backedge-taken count is SCEVCouldNotCompute. 1467 // Note that this serves two purposes: It filters out loops that are 1468 // simply not analyzable, and it covers the case where this code is 1469 // being called from within backedge-taken count analysis, such that 1470 // attempting to ask for the backedge-taken count would likely result 1471 // in infinite recursion. In the later case, the analysis code will 1472 // cope with a conservative value, and it will take care to purge 1473 // that value once it has finished. 1474 const SCEV *MaxBECount = getMaxBackedgeTakenCount(L); 1475 if (!isa<SCEVCouldNotCompute>(MaxBECount)) { 1476 // Manually compute the final value for AR, checking for 1477 // overflow. 1478 1479 // Check whether the backedge-taken count can be losslessly casted to 1480 // the addrec's type. The count is always unsigned. 1481 const SCEV *CastedMaxBECount = 1482 getTruncateOrZeroExtend(MaxBECount, Start->getType()); 1483 const SCEV *RecastedMaxBECount = 1484 getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType()); 1485 if (MaxBECount == RecastedMaxBECount) { 1486 Type *WideTy = IntegerType::get(getContext(), BitWidth * 2); 1487 // Check whether Start+Step*MaxBECount has no unsigned overflow. 1488 const SCEV *ZMul = getMulExpr(CastedMaxBECount, Step); 1489 const SCEV *ZAdd = getZeroExtendExpr(getAddExpr(Start, ZMul), WideTy); 1490 const SCEV *WideStart = getZeroExtendExpr(Start, WideTy); 1491 const SCEV *WideMaxBECount = 1492 getZeroExtendExpr(CastedMaxBECount, WideTy); 1493 const SCEV *OperandExtendedAdd = 1494 getAddExpr(WideStart, 1495 getMulExpr(WideMaxBECount, 1496 getZeroExtendExpr(Step, WideTy))); 1497 if (ZAdd == OperandExtendedAdd) { 1498 // Cache knowledge of AR NUW, which is propagated to this AddRec. 1499 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW); 1500 // Return the expression with the addrec on the outside. 1501 return getAddRecExpr( 1502 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this), 1503 getZeroExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1504 } 1505 // Similar to above, only this time treat the step value as signed. 1506 // This covers loops that count down. 1507 OperandExtendedAdd = 1508 getAddExpr(WideStart, 1509 getMulExpr(WideMaxBECount, 1510 getSignExtendExpr(Step, WideTy))); 1511 if (ZAdd == OperandExtendedAdd) { 1512 // Cache knowledge of AR NW, which is propagated to this AddRec. 1513 // Negative step causes unsigned wrap, but it still can't self-wrap. 1514 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW); 1515 // Return the expression with the addrec on the outside. 1516 return getAddRecExpr( 1517 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this), 1518 getSignExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1519 } 1520 } 1521 } 1522 1523 // Normally, in the cases we can prove no-overflow via a 1524 // backedge guarding condition, we can also compute a backedge 1525 // taken count for the loop. The exceptions are assumptions and 1526 // guards present in the loop -- SCEV is not great at exploiting 1527 // these to compute max backedge taken counts, but can still use 1528 // these to prove lack of overflow. Use this fact to avoid 1529 // doing extra work that may not pay off. 1530 if (!isa<SCEVCouldNotCompute>(MaxBECount) || HasGuards || 1531 !AC.assumptions().empty()) { 1532 // If the backedge is guarded by a comparison with the pre-inc 1533 // value the addrec is safe. Also, if the entry is guarded by 1534 // a comparison with the start value and the backedge is 1535 // guarded by a comparison with the post-inc value, the addrec 1536 // is safe. 1537 if (isKnownPositive(Step)) { 1538 const SCEV *N = getConstant(APInt::getMinValue(BitWidth) - 1539 getUnsignedRange(Step).getUnsignedMax()); 1540 if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT, AR, N) || 1541 (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_ULT, Start, N) && 1542 isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT, 1543 AR->getPostIncExpr(*this), N))) { 1544 // Cache knowledge of AR NUW, which is propagated to this 1545 // AddRec. 1546 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW); 1547 // Return the expression with the addrec on the outside. 1548 return getAddRecExpr( 1549 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this), 1550 getZeroExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1551 } 1552 } else if (isKnownNegative(Step)) { 1553 const SCEV *N = getConstant(APInt::getMaxValue(BitWidth) - 1554 getSignedRange(Step).getSignedMin()); 1555 if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, AR, N) || 1556 (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_UGT, Start, N) && 1557 isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, 1558 AR->getPostIncExpr(*this), N))) { 1559 // Cache knowledge of AR NW, which is propagated to this 1560 // AddRec. Negative step causes unsigned wrap, but it 1561 // still can't self-wrap. 1562 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW); 1563 // Return the expression with the addrec on the outside. 1564 return getAddRecExpr( 1565 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this), 1566 getSignExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1567 } 1568 } 1569 } 1570 1571 if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) { 1572 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW); 1573 return getAddRecExpr( 1574 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this), 1575 getZeroExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1576 } 1577 } 1578 1579 if (auto *SA = dyn_cast<SCEVAddExpr>(Op)) { 1580 // zext((A + B + ...)<nuw>) --> (zext(A) + zext(B) + ...)<nuw> 1581 if (SA->hasNoUnsignedWrap()) { 1582 // If the addition does not unsign overflow then we can, by definition, 1583 // commute the zero extension with the addition operation. 1584 SmallVector<const SCEV *, 4> Ops; 1585 for (const auto *Op : SA->operands()) 1586 Ops.push_back(getZeroExtendExpr(Op, Ty)); 1587 return getAddExpr(Ops, SCEV::FlagNUW); 1588 } 1589 } 1590 1591 // The cast wasn't folded; create an explicit cast node. 1592 // Recompute the insert position, as it may have been invalidated. 1593 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1594 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator), 1595 Op, Ty); 1596 UniqueSCEVs.InsertNode(S, IP); 1597 return S; 1598 } 1599 1600 const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op, 1601 Type *Ty) { 1602 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && 1603 "This is not an extending conversion!"); 1604 assert(isSCEVable(Ty) && 1605 "This is not a conversion to a SCEVable type!"); 1606 Ty = getEffectiveSCEVType(Ty); 1607 1608 // Fold if the operand is constant. 1609 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 1610 return getConstant( 1611 cast<ConstantInt>(ConstantExpr::getSExt(SC->getValue(), Ty))); 1612 1613 // sext(sext(x)) --> sext(x) 1614 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op)) 1615 return getSignExtendExpr(SS->getOperand(), Ty); 1616 1617 // sext(zext(x)) --> zext(x) 1618 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op)) 1619 return getZeroExtendExpr(SZ->getOperand(), Ty); 1620 1621 // Before doing any expensive analysis, check to see if we've already 1622 // computed a SCEV for this Op and Ty. 1623 FoldingSetNodeID ID; 1624 ID.AddInteger(scSignExtend); 1625 ID.AddPointer(Op); 1626 ID.AddPointer(Ty); 1627 void *IP = nullptr; 1628 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1629 1630 // sext(trunc(x)) --> sext(x) or x or trunc(x) 1631 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) { 1632 // It's possible the bits taken off by the truncate were all sign bits. If 1633 // so, we should be able to simplify this further. 1634 const SCEV *X = ST->getOperand(); 1635 ConstantRange CR = getSignedRange(X); 1636 unsigned TruncBits = getTypeSizeInBits(ST->getType()); 1637 unsigned NewBits = getTypeSizeInBits(Ty); 1638 if (CR.truncate(TruncBits).signExtend(NewBits).contains( 1639 CR.sextOrTrunc(NewBits))) 1640 return getTruncateOrSignExtend(X, Ty); 1641 } 1642 1643 // sext(C1 + (C2 * x)) --> C1 + sext(C2 * x) if C1 < C2 1644 if (auto *SA = dyn_cast<SCEVAddExpr>(Op)) { 1645 if (SA->getNumOperands() == 2) { 1646 auto *SC1 = dyn_cast<SCEVConstant>(SA->getOperand(0)); 1647 auto *SMul = dyn_cast<SCEVMulExpr>(SA->getOperand(1)); 1648 if (SMul && SC1) { 1649 if (auto *SC2 = dyn_cast<SCEVConstant>(SMul->getOperand(0))) { 1650 const APInt &C1 = SC1->getAPInt(); 1651 const APInt &C2 = SC2->getAPInt(); 1652 if (C1.isStrictlyPositive() && C2.isStrictlyPositive() && 1653 C2.ugt(C1) && C2.isPowerOf2()) 1654 return getAddExpr(getSignExtendExpr(SC1, Ty), 1655 getSignExtendExpr(SMul, Ty)); 1656 } 1657 } 1658 } 1659 1660 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw> 1661 if (SA->hasNoSignedWrap()) { 1662 // If the addition does not sign overflow then we can, by definition, 1663 // commute the sign extension with the addition operation. 1664 SmallVector<const SCEV *, 4> Ops; 1665 for (const auto *Op : SA->operands()) 1666 Ops.push_back(getSignExtendExpr(Op, Ty)); 1667 return getAddExpr(Ops, SCEV::FlagNSW); 1668 } 1669 } 1670 // If the input value is a chrec scev, and we can prove that the value 1671 // did not overflow the old, smaller, value, we can sign extend all of the 1672 // operands (often constants). This allows analysis of something like 1673 // this: for (signed char X = 0; X < 100; ++X) { int Y = X; } 1674 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) 1675 if (AR->isAffine()) { 1676 const SCEV *Start = AR->getStart(); 1677 const SCEV *Step = AR->getStepRecurrence(*this); 1678 unsigned BitWidth = getTypeSizeInBits(AR->getType()); 1679 const Loop *L = AR->getLoop(); 1680 1681 if (!AR->hasNoSignedWrap()) { 1682 auto NewFlags = proveNoWrapViaConstantRanges(AR); 1683 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(NewFlags); 1684 } 1685 1686 // If we have special knowledge that this addrec won't overflow, 1687 // we don't need to do any further analysis. 1688 if (AR->hasNoSignedWrap()) 1689 return getAddRecExpr( 1690 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this), 1691 getSignExtendExpr(Step, Ty), L, SCEV::FlagNSW); 1692 1693 // Check whether the backedge-taken count is SCEVCouldNotCompute. 1694 // Note that this serves two purposes: It filters out loops that are 1695 // simply not analyzable, and it covers the case where this code is 1696 // being called from within backedge-taken count analysis, such that 1697 // attempting to ask for the backedge-taken count would likely result 1698 // in infinite recursion. In the later case, the analysis code will 1699 // cope with a conservative value, and it will take care to purge 1700 // that value once it has finished. 1701 const SCEV *MaxBECount = getMaxBackedgeTakenCount(L); 1702 if (!isa<SCEVCouldNotCompute>(MaxBECount)) { 1703 // Manually compute the final value for AR, checking for 1704 // overflow. 1705 1706 // Check whether the backedge-taken count can be losslessly casted to 1707 // the addrec's type. The count is always unsigned. 1708 const SCEV *CastedMaxBECount = 1709 getTruncateOrZeroExtend(MaxBECount, Start->getType()); 1710 const SCEV *RecastedMaxBECount = 1711 getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType()); 1712 if (MaxBECount == RecastedMaxBECount) { 1713 Type *WideTy = IntegerType::get(getContext(), BitWidth * 2); 1714 // Check whether Start+Step*MaxBECount has no signed overflow. 1715 const SCEV *SMul = getMulExpr(CastedMaxBECount, Step); 1716 const SCEV *SAdd = getSignExtendExpr(getAddExpr(Start, SMul), WideTy); 1717 const SCEV *WideStart = getSignExtendExpr(Start, WideTy); 1718 const SCEV *WideMaxBECount = 1719 getZeroExtendExpr(CastedMaxBECount, WideTy); 1720 const SCEV *OperandExtendedAdd = 1721 getAddExpr(WideStart, 1722 getMulExpr(WideMaxBECount, 1723 getSignExtendExpr(Step, WideTy))); 1724 if (SAdd == OperandExtendedAdd) { 1725 // Cache knowledge of AR NSW, which is propagated to this AddRec. 1726 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW); 1727 // Return the expression with the addrec on the outside. 1728 return getAddRecExpr( 1729 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this), 1730 getSignExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1731 } 1732 // Similar to above, only this time treat the step value as unsigned. 1733 // This covers loops that count up with an unsigned step. 1734 OperandExtendedAdd = 1735 getAddExpr(WideStart, 1736 getMulExpr(WideMaxBECount, 1737 getZeroExtendExpr(Step, WideTy))); 1738 if (SAdd == OperandExtendedAdd) { 1739 // If AR wraps around then 1740 // 1741 // abs(Step) * MaxBECount > unsigned-max(AR->getType()) 1742 // => SAdd != OperandExtendedAdd 1743 // 1744 // Thus (AR is not NW => SAdd != OperandExtendedAdd) <=> 1745 // (SAdd == OperandExtendedAdd => AR is NW) 1746 1747 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW); 1748 1749 // Return the expression with the addrec on the outside. 1750 return getAddRecExpr( 1751 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this), 1752 getZeroExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1753 } 1754 } 1755 } 1756 1757 // Normally, in the cases we can prove no-overflow via a 1758 // backedge guarding condition, we can also compute a backedge 1759 // taken count for the loop. The exceptions are assumptions and 1760 // guards present in the loop -- SCEV is not great at exploiting 1761 // these to compute max backedge taken counts, but can still use 1762 // these to prove lack of overflow. Use this fact to avoid 1763 // doing extra work that may not pay off. 1764 1765 if (!isa<SCEVCouldNotCompute>(MaxBECount) || HasGuards || 1766 !AC.assumptions().empty()) { 1767 // If the backedge is guarded by a comparison with the pre-inc 1768 // value the addrec is safe. Also, if the entry is guarded by 1769 // a comparison with the start value and the backedge is 1770 // guarded by a comparison with the post-inc value, the addrec 1771 // is safe. 1772 ICmpInst::Predicate Pred; 1773 const SCEV *OverflowLimit = 1774 getSignedOverflowLimitForStep(Step, &Pred, this); 1775 if (OverflowLimit && 1776 (isLoopBackedgeGuardedByCond(L, Pred, AR, OverflowLimit) || 1777 (isLoopEntryGuardedByCond(L, Pred, Start, OverflowLimit) && 1778 isLoopBackedgeGuardedByCond(L, Pred, AR->getPostIncExpr(*this), 1779 OverflowLimit)))) { 1780 // Cache knowledge of AR NSW, then propagate NSW to the wide AddRec. 1781 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW); 1782 return getAddRecExpr( 1783 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this), 1784 getSignExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1785 } 1786 } 1787 1788 // If Start and Step are constants, check if we can apply this 1789 // transformation: 1790 // sext{C1,+,C2} --> C1 + sext{0,+,C2} if C1 < C2 1791 auto *SC1 = dyn_cast<SCEVConstant>(Start); 1792 auto *SC2 = dyn_cast<SCEVConstant>(Step); 1793 if (SC1 && SC2) { 1794 const APInt &C1 = SC1->getAPInt(); 1795 const APInt &C2 = SC2->getAPInt(); 1796 if (C1.isStrictlyPositive() && C2.isStrictlyPositive() && C2.ugt(C1) && 1797 C2.isPowerOf2()) { 1798 Start = getSignExtendExpr(Start, Ty); 1799 const SCEV *NewAR = getAddRecExpr(getZero(AR->getType()), Step, L, 1800 AR->getNoWrapFlags()); 1801 return getAddExpr(Start, getSignExtendExpr(NewAR, Ty)); 1802 } 1803 } 1804 1805 if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) { 1806 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW); 1807 return getAddRecExpr( 1808 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this), 1809 getSignExtendExpr(Step, Ty), L, AR->getNoWrapFlags()); 1810 } 1811 } 1812 1813 // If the input value is provably positive and we could not simplify 1814 // away the sext build a zext instead. 1815 if (isKnownNonNegative(Op)) 1816 return getZeroExtendExpr(Op, Ty); 1817 1818 // The cast wasn't folded; create an explicit cast node. 1819 // Recompute the insert position, as it may have been invalidated. 1820 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1821 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator), 1822 Op, Ty); 1823 UniqueSCEVs.InsertNode(S, IP); 1824 return S; 1825 } 1826 1827 /// getAnyExtendExpr - Return a SCEV for the given operand extended with 1828 /// unspecified bits out to the given type. 1829 /// 1830 const SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op, 1831 Type *Ty) { 1832 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && 1833 "This is not an extending conversion!"); 1834 assert(isSCEVable(Ty) && 1835 "This is not a conversion to a SCEVable type!"); 1836 Ty = getEffectiveSCEVType(Ty); 1837 1838 // Sign-extend negative constants. 1839 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 1840 if (SC->getAPInt().isNegative()) 1841 return getSignExtendExpr(Op, Ty); 1842 1843 // Peel off a truncate cast. 1844 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Op)) { 1845 const SCEV *NewOp = T->getOperand(); 1846 if (getTypeSizeInBits(NewOp->getType()) < getTypeSizeInBits(Ty)) 1847 return getAnyExtendExpr(NewOp, Ty); 1848 return getTruncateOrNoop(NewOp, Ty); 1849 } 1850 1851 // Next try a zext cast. If the cast is folded, use it. 1852 const SCEV *ZExt = getZeroExtendExpr(Op, Ty); 1853 if (!isa<SCEVZeroExtendExpr>(ZExt)) 1854 return ZExt; 1855 1856 // Next try a sext cast. If the cast is folded, use it. 1857 const SCEV *SExt = getSignExtendExpr(Op, Ty); 1858 if (!isa<SCEVSignExtendExpr>(SExt)) 1859 return SExt; 1860 1861 // Force the cast to be folded into the operands of an addrec. 1862 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) { 1863 SmallVector<const SCEV *, 4> Ops; 1864 for (const SCEV *Op : AR->operands()) 1865 Ops.push_back(getAnyExtendExpr(Op, Ty)); 1866 return getAddRecExpr(Ops, AR->getLoop(), SCEV::FlagNW); 1867 } 1868 1869 // If the expression is obviously signed, use the sext cast value. 1870 if (isa<SCEVSMaxExpr>(Op)) 1871 return SExt; 1872 1873 // Absent any other information, use the zext cast value. 1874 return ZExt; 1875 } 1876 1877 /// Process the given Ops list, which is a list of operands to be added under 1878 /// the given scale, update the given map. This is a helper function for 1879 /// getAddRecExpr. As an example of what it does, given a sequence of operands 1880 /// that would form an add expression like this: 1881 /// 1882 /// m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r) 1883 /// 1884 /// where A and B are constants, update the map with these values: 1885 /// 1886 /// (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0) 1887 /// 1888 /// and add 13 + A*B*29 to AccumulatedConstant. 1889 /// This will allow getAddRecExpr to produce this: 1890 /// 1891 /// 13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B) 1892 /// 1893 /// This form often exposes folding opportunities that are hidden in 1894 /// the original operand list. 1895 /// 1896 /// Return true iff it appears that any interesting folding opportunities 1897 /// may be exposed. This helps getAddRecExpr short-circuit extra work in 1898 /// the common case where no interesting opportunities are present, and 1899 /// is also used as a check to avoid infinite recursion. 1900 /// 1901 static bool 1902 CollectAddOperandsWithScales(DenseMap<const SCEV *, APInt> &M, 1903 SmallVectorImpl<const SCEV *> &NewOps, 1904 APInt &AccumulatedConstant, 1905 const SCEV *const *Ops, size_t NumOperands, 1906 const APInt &Scale, 1907 ScalarEvolution &SE) { 1908 bool Interesting = false; 1909 1910 // Iterate over the add operands. They are sorted, with constants first. 1911 unsigned i = 0; 1912 while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) { 1913 ++i; 1914 // Pull a buried constant out to the outside. 1915 if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero()) 1916 Interesting = true; 1917 AccumulatedConstant += Scale * C->getAPInt(); 1918 } 1919 1920 // Next comes everything else. We're especially interested in multiplies 1921 // here, but they're in the middle, so just visit the rest with one loop. 1922 for (; i != NumOperands; ++i) { 1923 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[i]); 1924 if (Mul && isa<SCEVConstant>(Mul->getOperand(0))) { 1925 APInt NewScale = 1926 Scale * cast<SCEVConstant>(Mul->getOperand(0))->getAPInt(); 1927 if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Mul->getOperand(1))) { 1928 // A multiplication of a constant with another add; recurse. 1929 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Mul->getOperand(1)); 1930 Interesting |= 1931 CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant, 1932 Add->op_begin(), Add->getNumOperands(), 1933 NewScale, SE); 1934 } else { 1935 // A multiplication of a constant with some other value. Update 1936 // the map. 1937 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin()+1, Mul->op_end()); 1938 const SCEV *Key = SE.getMulExpr(MulOps); 1939 auto Pair = M.insert({Key, NewScale}); 1940 if (Pair.second) { 1941 NewOps.push_back(Pair.first->first); 1942 } else { 1943 Pair.first->second += NewScale; 1944 // The map already had an entry for this value, which may indicate 1945 // a folding opportunity. 1946 Interesting = true; 1947 } 1948 } 1949 } else { 1950 // An ordinary operand. Update the map. 1951 std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair = 1952 M.insert({Ops[i], Scale}); 1953 if (Pair.second) { 1954 NewOps.push_back(Pair.first->first); 1955 } else { 1956 Pair.first->second += Scale; 1957 // The map already had an entry for this value, which may indicate 1958 // a folding opportunity. 1959 Interesting = true; 1960 } 1961 } 1962 } 1963 1964 return Interesting; 1965 } 1966 1967 // We're trying to construct a SCEV of type `Type' with `Ops' as operands and 1968 // `OldFlags' as can't-wrap behavior. Infer a more aggressive set of 1969 // can't-overflow flags for the operation if possible. 1970 static SCEV::NoWrapFlags 1971 StrengthenNoWrapFlags(ScalarEvolution *SE, SCEVTypes Type, 1972 const SmallVectorImpl<const SCEV *> &Ops, 1973 SCEV::NoWrapFlags Flags) { 1974 using namespace std::placeholders; 1975 typedef OverflowingBinaryOperator OBO; 1976 1977 bool CanAnalyze = 1978 Type == scAddExpr || Type == scAddRecExpr || Type == scMulExpr; 1979 (void)CanAnalyze; 1980 assert(CanAnalyze && "don't call from other places!"); 1981 1982 int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW; 1983 SCEV::NoWrapFlags SignOrUnsignWrap = 1984 ScalarEvolution::maskFlags(Flags, SignOrUnsignMask); 1985 1986 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW. 1987 auto IsKnownNonNegative = [&](const SCEV *S) { 1988 return SE->isKnownNonNegative(S); 1989 }; 1990 1991 if (SignOrUnsignWrap == SCEV::FlagNSW && all_of(Ops, IsKnownNonNegative)) 1992 Flags = 1993 ScalarEvolution::setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask); 1994 1995 SignOrUnsignWrap = ScalarEvolution::maskFlags(Flags, SignOrUnsignMask); 1996 1997 if (SignOrUnsignWrap != SignOrUnsignMask && Type == scAddExpr && 1998 Ops.size() == 2 && isa<SCEVConstant>(Ops[0])) { 1999 2000 // (A + C) --> (A + C)<nsw> if the addition does not sign overflow 2001 // (A + C) --> (A + C)<nuw> if the addition does not unsign overflow 2002 2003 const APInt &C = cast<SCEVConstant>(Ops[0])->getAPInt(); 2004 if (!(SignOrUnsignWrap & SCEV::FlagNSW)) { 2005 auto NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion( 2006 Instruction::Add, C, OBO::NoSignedWrap); 2007 if (NSWRegion.contains(SE->getSignedRange(Ops[1]))) 2008 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNSW); 2009 } 2010 if (!(SignOrUnsignWrap & SCEV::FlagNUW)) { 2011 auto NUWRegion = ConstantRange::makeGuaranteedNoWrapRegion( 2012 Instruction::Add, C, OBO::NoUnsignedWrap); 2013 if (NUWRegion.contains(SE->getUnsignedRange(Ops[1]))) 2014 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW); 2015 } 2016 } 2017 2018 return Flags; 2019 } 2020 2021 /// Get a canonical add expression, or something simpler if possible. 2022 const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops, 2023 SCEV::NoWrapFlags Flags) { 2024 assert(!(Flags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) && 2025 "only nuw or nsw allowed"); 2026 assert(!Ops.empty() && "Cannot get empty add!"); 2027 if (Ops.size() == 1) return Ops[0]; 2028 #ifndef NDEBUG 2029 Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 2030 for (unsigned i = 1, e = Ops.size(); i != e; ++i) 2031 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 2032 "SCEVAddExpr operand types don't match!"); 2033 #endif 2034 2035 // Sort by complexity, this groups all similar expression types together. 2036 GroupByComplexity(Ops, &LI); 2037 2038 Flags = StrengthenNoWrapFlags(this, scAddExpr, Ops, Flags); 2039 2040 // If there are any constants, fold them together. 2041 unsigned Idx = 0; 2042 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 2043 ++Idx; 2044 assert(Idx < Ops.size()); 2045 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 2046 // We found two constants, fold them together! 2047 Ops[0] = getConstant(LHSC->getAPInt() + RHSC->getAPInt()); 2048 if (Ops.size() == 2) return Ops[0]; 2049 Ops.erase(Ops.begin()+1); // Erase the folded element 2050 LHSC = cast<SCEVConstant>(Ops[0]); 2051 } 2052 2053 // If we are left with a constant zero being added, strip it off. 2054 if (LHSC->getValue()->isZero()) { 2055 Ops.erase(Ops.begin()); 2056 --Idx; 2057 } 2058 2059 if (Ops.size() == 1) return Ops[0]; 2060 } 2061 2062 // Okay, check to see if the same value occurs in the operand list more than 2063 // once. If so, merge them together into an multiply expression. Since we 2064 // sorted the list, these values are required to be adjacent. 2065 Type *Ty = Ops[0]->getType(); 2066 bool FoundMatch = false; 2067 for (unsigned i = 0, e = Ops.size(); i != e-1; ++i) 2068 if (Ops[i] == Ops[i+1]) { // X + Y + Y --> X + Y*2 2069 // Scan ahead to count how many equal operands there are. 2070 unsigned Count = 2; 2071 while (i+Count != e && Ops[i+Count] == Ops[i]) 2072 ++Count; 2073 // Merge the values into a multiply. 2074 const SCEV *Scale = getConstant(Ty, Count); 2075 const SCEV *Mul = getMulExpr(Scale, Ops[i]); 2076 if (Ops.size() == Count) 2077 return Mul; 2078 Ops[i] = Mul; 2079 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+Count); 2080 --i; e -= Count - 1; 2081 FoundMatch = true; 2082 } 2083 if (FoundMatch) 2084 return getAddExpr(Ops, Flags); 2085 2086 // Check for truncates. If all the operands are truncated from the same 2087 // type, see if factoring out the truncate would permit the result to be 2088 // folded. eg., trunc(x) + m*trunc(n) --> trunc(x + trunc(m)*n) 2089 // if the contents of the resulting outer trunc fold to something simple. 2090 for (; Idx < Ops.size() && isa<SCEVTruncateExpr>(Ops[Idx]); ++Idx) { 2091 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Ops[Idx]); 2092 Type *DstType = Trunc->getType(); 2093 Type *SrcType = Trunc->getOperand()->getType(); 2094 SmallVector<const SCEV *, 8> LargeOps; 2095 bool Ok = true; 2096 // Check all the operands to see if they can be represented in the 2097 // source type of the truncate. 2098 for (unsigned i = 0, e = Ops.size(); i != e; ++i) { 2099 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Ops[i])) { 2100 if (T->getOperand()->getType() != SrcType) { 2101 Ok = false; 2102 break; 2103 } 2104 LargeOps.push_back(T->getOperand()); 2105 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) { 2106 LargeOps.push_back(getAnyExtendExpr(C, SrcType)); 2107 } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Ops[i])) { 2108 SmallVector<const SCEV *, 8> LargeMulOps; 2109 for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) { 2110 if (const SCEVTruncateExpr *T = 2111 dyn_cast<SCEVTruncateExpr>(M->getOperand(j))) { 2112 if (T->getOperand()->getType() != SrcType) { 2113 Ok = false; 2114 break; 2115 } 2116 LargeMulOps.push_back(T->getOperand()); 2117 } else if (const auto *C = dyn_cast<SCEVConstant>(M->getOperand(j))) { 2118 LargeMulOps.push_back(getAnyExtendExpr(C, SrcType)); 2119 } else { 2120 Ok = false; 2121 break; 2122 } 2123 } 2124 if (Ok) 2125 LargeOps.push_back(getMulExpr(LargeMulOps)); 2126 } else { 2127 Ok = false; 2128 break; 2129 } 2130 } 2131 if (Ok) { 2132 // Evaluate the expression in the larger type. 2133 const SCEV *Fold = getAddExpr(LargeOps, Flags); 2134 // If it folds to something simple, use it. Otherwise, don't. 2135 if (isa<SCEVConstant>(Fold) || isa<SCEVUnknown>(Fold)) 2136 return getTruncateExpr(Fold, DstType); 2137 } 2138 } 2139 2140 // Skip past any other cast SCEVs. 2141 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr) 2142 ++Idx; 2143 2144 // If there are add operands they would be next. 2145 if (Idx < Ops.size()) { 2146 bool DeletedAdd = false; 2147 while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[Idx])) { 2148 // If we have an add, expand the add operands onto the end of the operands 2149 // list. 2150 Ops.erase(Ops.begin()+Idx); 2151 Ops.append(Add->op_begin(), Add->op_end()); 2152 DeletedAdd = true; 2153 } 2154 2155 // If we deleted at least one add, we added operands to the end of the list, 2156 // and they are not necessarily sorted. Recurse to resort and resimplify 2157 // any operands we just acquired. 2158 if (DeletedAdd) 2159 return getAddExpr(Ops); 2160 } 2161 2162 // Skip over the add expression until we get to a multiply. 2163 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr) 2164 ++Idx; 2165 2166 // Check to see if there are any folding opportunities present with 2167 // operands multiplied by constant values. 2168 if (Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx])) { 2169 uint64_t BitWidth = getTypeSizeInBits(Ty); 2170 DenseMap<const SCEV *, APInt> M; 2171 SmallVector<const SCEV *, 8> NewOps; 2172 APInt AccumulatedConstant(BitWidth, 0); 2173 if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant, 2174 Ops.data(), Ops.size(), 2175 APInt(BitWidth, 1), *this)) { 2176 struct APIntCompare { 2177 bool operator()(const APInt &LHS, const APInt &RHS) const { 2178 return LHS.ult(RHS); 2179 } 2180 }; 2181 2182 // Some interesting folding opportunity is present, so its worthwhile to 2183 // re-generate the operands list. Group the operands by constant scale, 2184 // to avoid multiplying by the same constant scale multiple times. 2185 std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare> MulOpLists; 2186 for (const SCEV *NewOp : NewOps) 2187 MulOpLists[M.find(NewOp)->second].push_back(NewOp); 2188 // Re-generate the operands list. 2189 Ops.clear(); 2190 if (AccumulatedConstant != 0) 2191 Ops.push_back(getConstant(AccumulatedConstant)); 2192 for (auto &MulOp : MulOpLists) 2193 if (MulOp.first != 0) 2194 Ops.push_back(getMulExpr(getConstant(MulOp.first), 2195 getAddExpr(MulOp.second))); 2196 if (Ops.empty()) 2197 return getZero(Ty); 2198 if (Ops.size() == 1) 2199 return Ops[0]; 2200 return getAddExpr(Ops); 2201 } 2202 } 2203 2204 // If we are adding something to a multiply expression, make sure the 2205 // something is not already an operand of the multiply. If so, merge it into 2206 // the multiply. 2207 for (; Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx]); ++Idx) { 2208 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Ops[Idx]); 2209 for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) { 2210 const SCEV *MulOpSCEV = Mul->getOperand(MulOp); 2211 if (isa<SCEVConstant>(MulOpSCEV)) 2212 continue; 2213 for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp) 2214 if (MulOpSCEV == Ops[AddOp]) { 2215 // Fold W + X + (X * Y * Z) --> W + (X * ((Y*Z)+1)) 2216 const SCEV *InnerMul = Mul->getOperand(MulOp == 0); 2217 if (Mul->getNumOperands() != 2) { 2218 // If the multiply has more than two operands, we must get the 2219 // Y*Z term. 2220 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(), 2221 Mul->op_begin()+MulOp); 2222 MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end()); 2223 InnerMul = getMulExpr(MulOps); 2224 } 2225 const SCEV *One = getOne(Ty); 2226 const SCEV *AddOne = getAddExpr(One, InnerMul); 2227 const SCEV *OuterMul = getMulExpr(AddOne, MulOpSCEV); 2228 if (Ops.size() == 2) return OuterMul; 2229 if (AddOp < Idx) { 2230 Ops.erase(Ops.begin()+AddOp); 2231 Ops.erase(Ops.begin()+Idx-1); 2232 } else { 2233 Ops.erase(Ops.begin()+Idx); 2234 Ops.erase(Ops.begin()+AddOp-1); 2235 } 2236 Ops.push_back(OuterMul); 2237 return getAddExpr(Ops); 2238 } 2239 2240 // Check this multiply against other multiplies being added together. 2241 for (unsigned OtherMulIdx = Idx+1; 2242 OtherMulIdx < Ops.size() && isa<SCEVMulExpr>(Ops[OtherMulIdx]); 2243 ++OtherMulIdx) { 2244 const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Ops[OtherMulIdx]); 2245 // If MulOp occurs in OtherMul, we can fold the two multiplies 2246 // together. 2247 for (unsigned OMulOp = 0, e = OtherMul->getNumOperands(); 2248 OMulOp != e; ++OMulOp) 2249 if (OtherMul->getOperand(OMulOp) == MulOpSCEV) { 2250 // Fold X + (A*B*C) + (A*D*E) --> X + (A*(B*C+D*E)) 2251 const SCEV *InnerMul1 = Mul->getOperand(MulOp == 0); 2252 if (Mul->getNumOperands() != 2) { 2253 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(), 2254 Mul->op_begin()+MulOp); 2255 MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end()); 2256 InnerMul1 = getMulExpr(MulOps); 2257 } 2258 const SCEV *InnerMul2 = OtherMul->getOperand(OMulOp == 0); 2259 if (OtherMul->getNumOperands() != 2) { 2260 SmallVector<const SCEV *, 4> MulOps(OtherMul->op_begin(), 2261 OtherMul->op_begin()+OMulOp); 2262 MulOps.append(OtherMul->op_begin()+OMulOp+1, OtherMul->op_end()); 2263 InnerMul2 = getMulExpr(MulOps); 2264 } 2265 const SCEV *InnerMulSum = getAddExpr(InnerMul1,InnerMul2); 2266 const SCEV *OuterMul = getMulExpr(MulOpSCEV, InnerMulSum); 2267 if (Ops.size() == 2) return OuterMul; 2268 Ops.erase(Ops.begin()+Idx); 2269 Ops.erase(Ops.begin()+OtherMulIdx-1); 2270 Ops.push_back(OuterMul); 2271 return getAddExpr(Ops); 2272 } 2273 } 2274 } 2275 } 2276 2277 // If there are any add recurrences in the operands list, see if any other 2278 // added values are loop invariant. If so, we can fold them into the 2279 // recurrence. 2280 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr) 2281 ++Idx; 2282 2283 // Scan over all recurrences, trying to fold loop invariants into them. 2284 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) { 2285 // Scan all of the other operands to this add and add them to the vector if 2286 // they are loop invariant w.r.t. the recurrence. 2287 SmallVector<const SCEV *, 8> LIOps; 2288 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]); 2289 const Loop *AddRecLoop = AddRec->getLoop(); 2290 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2291 if (isLoopInvariant(Ops[i], AddRecLoop)) { 2292 LIOps.push_back(Ops[i]); 2293 Ops.erase(Ops.begin()+i); 2294 --i; --e; 2295 } 2296 2297 // If we found some loop invariants, fold them into the recurrence. 2298 if (!LIOps.empty()) { 2299 // NLI + LI + {Start,+,Step} --> NLI + {LI+Start,+,Step} 2300 LIOps.push_back(AddRec->getStart()); 2301 2302 SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(), 2303 AddRec->op_end()); 2304 // This follows from the fact that the no-wrap flags on the outer add 2305 // expression are applicable on the 0th iteration, when the add recurrence 2306 // will be equal to its start value. 2307 AddRecOps[0] = getAddExpr(LIOps, Flags); 2308 2309 // Build the new addrec. Propagate the NUW and NSW flags if both the 2310 // outer add and the inner addrec are guaranteed to have no overflow. 2311 // Always propagate NW. 2312 Flags = AddRec->getNoWrapFlags(setFlags(Flags, SCEV::FlagNW)); 2313 const SCEV *NewRec = getAddRecExpr(AddRecOps, AddRecLoop, Flags); 2314 2315 // If all of the other operands were loop invariant, we are done. 2316 if (Ops.size() == 1) return NewRec; 2317 2318 // Otherwise, add the folded AddRec by the non-invariant parts. 2319 for (unsigned i = 0;; ++i) 2320 if (Ops[i] == AddRec) { 2321 Ops[i] = NewRec; 2322 break; 2323 } 2324 return getAddExpr(Ops); 2325 } 2326 2327 // Okay, if there weren't any loop invariants to be folded, check to see if 2328 // there are multiple AddRec's with the same loop induction variable being 2329 // added together. If so, we can fold them. 2330 for (unsigned OtherIdx = Idx+1; 2331 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 2332 ++OtherIdx) 2333 if (AddRecLoop == cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()) { 2334 // Other + {A,+,B}<L> + {C,+,D}<L> --> Other + {A+C,+,B+D}<L> 2335 SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(), 2336 AddRec->op_end()); 2337 for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 2338 ++OtherIdx) 2339 if (const auto *OtherAddRec = dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx])) 2340 if (OtherAddRec->getLoop() == AddRecLoop) { 2341 for (unsigned i = 0, e = OtherAddRec->getNumOperands(); 2342 i != e; ++i) { 2343 if (i >= AddRecOps.size()) { 2344 AddRecOps.append(OtherAddRec->op_begin()+i, 2345 OtherAddRec->op_end()); 2346 break; 2347 } 2348 AddRecOps[i] = getAddExpr(AddRecOps[i], 2349 OtherAddRec->getOperand(i)); 2350 } 2351 Ops.erase(Ops.begin() + OtherIdx); --OtherIdx; 2352 } 2353 // Step size has changed, so we cannot guarantee no self-wraparound. 2354 Ops[Idx] = getAddRecExpr(AddRecOps, AddRecLoop, SCEV::FlagAnyWrap); 2355 return getAddExpr(Ops); 2356 } 2357 2358 // Otherwise couldn't fold anything into this recurrence. Move onto the 2359 // next one. 2360 } 2361 2362 // Okay, it looks like we really DO need an add expr. Check to see if we 2363 // already have one, otherwise create a new one. 2364 FoldingSetNodeID ID; 2365 ID.AddInteger(scAddExpr); 2366 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2367 ID.AddPointer(Ops[i]); 2368 void *IP = nullptr; 2369 SCEVAddExpr *S = 2370 static_cast<SCEVAddExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 2371 if (!S) { 2372 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 2373 std::uninitialized_copy(Ops.begin(), Ops.end(), O); 2374 S = new (SCEVAllocator) SCEVAddExpr(ID.Intern(SCEVAllocator), 2375 O, Ops.size()); 2376 UniqueSCEVs.InsertNode(S, IP); 2377 } 2378 S->setNoWrapFlags(Flags); 2379 return S; 2380 } 2381 2382 static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) { 2383 uint64_t k = i*j; 2384 if (j > 1 && k / j != i) Overflow = true; 2385 return k; 2386 } 2387 2388 /// Compute the result of "n choose k", the binomial coefficient. If an 2389 /// intermediate computation overflows, Overflow will be set and the return will 2390 /// be garbage. Overflow is not cleared on absence of overflow. 2391 static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) { 2392 // We use the multiplicative formula: 2393 // n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 . 2394 // At each iteration, we take the n-th term of the numeral and divide by the 2395 // (k-n)th term of the denominator. This division will always produce an 2396 // integral result, and helps reduce the chance of overflow in the 2397 // intermediate computations. However, we can still overflow even when the 2398 // final result would fit. 2399 2400 if (n == 0 || n == k) return 1; 2401 if (k > n) return 0; 2402 2403 if (k > n/2) 2404 k = n-k; 2405 2406 uint64_t r = 1; 2407 for (uint64_t i = 1; i <= k; ++i) { 2408 r = umul_ov(r, n-(i-1), Overflow); 2409 r /= i; 2410 } 2411 return r; 2412 } 2413 2414 /// Determine if any of the operands in this SCEV are a constant or if 2415 /// any of the add or multiply expressions in this SCEV contain a constant. 2416 static bool containsConstantSomewhere(const SCEV *StartExpr) { 2417 SmallVector<const SCEV *, 4> Ops; 2418 Ops.push_back(StartExpr); 2419 while (!Ops.empty()) { 2420 const SCEV *CurrentExpr = Ops.pop_back_val(); 2421 if (isa<SCEVConstant>(*CurrentExpr)) 2422 return true; 2423 2424 if (isa<SCEVAddExpr>(*CurrentExpr) || isa<SCEVMulExpr>(*CurrentExpr)) { 2425 const auto *CurrentNAry = cast<SCEVNAryExpr>(CurrentExpr); 2426 Ops.append(CurrentNAry->op_begin(), CurrentNAry->op_end()); 2427 } 2428 } 2429 return false; 2430 } 2431 2432 /// Get a canonical multiply expression, or something simpler if possible. 2433 const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops, 2434 SCEV::NoWrapFlags Flags) { 2435 assert(Flags == maskFlags(Flags, SCEV::FlagNUW | SCEV::FlagNSW) && 2436 "only nuw or nsw allowed"); 2437 assert(!Ops.empty() && "Cannot get empty mul!"); 2438 if (Ops.size() == 1) return Ops[0]; 2439 #ifndef NDEBUG 2440 Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 2441 for (unsigned i = 1, e = Ops.size(); i != e; ++i) 2442 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 2443 "SCEVMulExpr operand types don't match!"); 2444 #endif 2445 2446 // Sort by complexity, this groups all similar expression types together. 2447 GroupByComplexity(Ops, &LI); 2448 2449 Flags = StrengthenNoWrapFlags(this, scMulExpr, Ops, Flags); 2450 2451 // If there are any constants, fold them together. 2452 unsigned Idx = 0; 2453 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 2454 2455 // C1*(C2+V) -> C1*C2 + C1*V 2456 if (Ops.size() == 2) 2457 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) 2458 // If any of Add's ops are Adds or Muls with a constant, 2459 // apply this transformation as well. 2460 if (Add->getNumOperands() == 2) 2461 if (containsConstantSomewhere(Add)) 2462 return getAddExpr(getMulExpr(LHSC, Add->getOperand(0)), 2463 getMulExpr(LHSC, Add->getOperand(1))); 2464 2465 ++Idx; 2466 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 2467 // We found two constants, fold them together! 2468 ConstantInt *Fold = 2469 ConstantInt::get(getContext(), LHSC->getAPInt() * RHSC->getAPInt()); 2470 Ops[0] = getConstant(Fold); 2471 Ops.erase(Ops.begin()+1); // Erase the folded element 2472 if (Ops.size() == 1) return Ops[0]; 2473 LHSC = cast<SCEVConstant>(Ops[0]); 2474 } 2475 2476 // If we are left with a constant one being multiplied, strip it off. 2477 if (cast<SCEVConstant>(Ops[0])->getValue()->equalsInt(1)) { 2478 Ops.erase(Ops.begin()); 2479 --Idx; 2480 } else if (cast<SCEVConstant>(Ops[0])->getValue()->isZero()) { 2481 // If we have a multiply of zero, it will always be zero. 2482 return Ops[0]; 2483 } else if (Ops[0]->isAllOnesValue()) { 2484 // If we have a mul by -1 of an add, try distributing the -1 among the 2485 // add operands. 2486 if (Ops.size() == 2) { 2487 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) { 2488 SmallVector<const SCEV *, 4> NewOps; 2489 bool AnyFolded = false; 2490 for (const SCEV *AddOp : Add->operands()) { 2491 const SCEV *Mul = getMulExpr(Ops[0], AddOp); 2492 if (!isa<SCEVMulExpr>(Mul)) AnyFolded = true; 2493 NewOps.push_back(Mul); 2494 } 2495 if (AnyFolded) 2496 return getAddExpr(NewOps); 2497 } else if (const auto *AddRec = dyn_cast<SCEVAddRecExpr>(Ops[1])) { 2498 // Negation preserves a recurrence's no self-wrap property. 2499 SmallVector<const SCEV *, 4> Operands; 2500 for (const SCEV *AddRecOp : AddRec->operands()) 2501 Operands.push_back(getMulExpr(Ops[0], AddRecOp)); 2502 2503 return getAddRecExpr(Operands, AddRec->getLoop(), 2504 AddRec->getNoWrapFlags(SCEV::FlagNW)); 2505 } 2506 } 2507 } 2508 2509 if (Ops.size() == 1) 2510 return Ops[0]; 2511 } 2512 2513 // Skip over the add expression until we get to a multiply. 2514 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr) 2515 ++Idx; 2516 2517 // If there are mul operands inline them all into this expression. 2518 if (Idx < Ops.size()) { 2519 bool DeletedMul = false; 2520 while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) { 2521 // If we have an mul, expand the mul operands onto the end of the operands 2522 // list. 2523 Ops.erase(Ops.begin()+Idx); 2524 Ops.append(Mul->op_begin(), Mul->op_end()); 2525 DeletedMul = true; 2526 } 2527 2528 // If we deleted at least one mul, we added operands to the end of the list, 2529 // and they are not necessarily sorted. Recurse to resort and resimplify 2530 // any operands we just acquired. 2531 if (DeletedMul) 2532 return getMulExpr(Ops); 2533 } 2534 2535 // If there are any add recurrences in the operands list, see if any other 2536 // added values are loop invariant. If so, we can fold them into the 2537 // recurrence. 2538 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr) 2539 ++Idx; 2540 2541 // Scan over all recurrences, trying to fold loop invariants into them. 2542 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) { 2543 // Scan all of the other operands to this mul and add them to the vector if 2544 // they are loop invariant w.r.t. the recurrence. 2545 SmallVector<const SCEV *, 8> LIOps; 2546 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]); 2547 const Loop *AddRecLoop = AddRec->getLoop(); 2548 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2549 if (isLoopInvariant(Ops[i], AddRecLoop)) { 2550 LIOps.push_back(Ops[i]); 2551 Ops.erase(Ops.begin()+i); 2552 --i; --e; 2553 } 2554 2555 // If we found some loop invariants, fold them into the recurrence. 2556 if (!LIOps.empty()) { 2557 // NLI * LI * {Start,+,Step} --> NLI * {LI*Start,+,LI*Step} 2558 SmallVector<const SCEV *, 4> NewOps; 2559 NewOps.reserve(AddRec->getNumOperands()); 2560 const SCEV *Scale = getMulExpr(LIOps); 2561 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) 2562 NewOps.push_back(getMulExpr(Scale, AddRec->getOperand(i))); 2563 2564 // Build the new addrec. Propagate the NUW and NSW flags if both the 2565 // outer mul and the inner addrec are guaranteed to have no overflow. 2566 // 2567 // No self-wrap cannot be guaranteed after changing the step size, but 2568 // will be inferred if either NUW or NSW is true. 2569 Flags = AddRec->getNoWrapFlags(clearFlags(Flags, SCEV::FlagNW)); 2570 const SCEV *NewRec = getAddRecExpr(NewOps, AddRecLoop, Flags); 2571 2572 // If all of the other operands were loop invariant, we are done. 2573 if (Ops.size() == 1) return NewRec; 2574 2575 // Otherwise, multiply the folded AddRec by the non-invariant parts. 2576 for (unsigned i = 0;; ++i) 2577 if (Ops[i] == AddRec) { 2578 Ops[i] = NewRec; 2579 break; 2580 } 2581 return getMulExpr(Ops); 2582 } 2583 2584 // Okay, if there weren't any loop invariants to be folded, check to see if 2585 // there are multiple AddRec's with the same loop induction variable being 2586 // multiplied together. If so, we can fold them. 2587 2588 // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L> 2589 // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [ 2590 // choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z 2591 // ]]],+,...up to x=2n}. 2592 // Note that the arguments to choose() are always integers with values 2593 // known at compile time, never SCEV objects. 2594 // 2595 // The implementation avoids pointless extra computations when the two 2596 // addrec's are of different length (mathematically, it's equivalent to 2597 // an infinite stream of zeros on the right). 2598 bool OpsModified = false; 2599 for (unsigned OtherIdx = Idx+1; 2600 OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 2601 ++OtherIdx) { 2602 const SCEVAddRecExpr *OtherAddRec = 2603 dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]); 2604 if (!OtherAddRec || OtherAddRec->getLoop() != AddRecLoop) 2605 continue; 2606 2607 bool Overflow = false; 2608 Type *Ty = AddRec->getType(); 2609 bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64; 2610 SmallVector<const SCEV*, 7> AddRecOps; 2611 for (int x = 0, xe = AddRec->getNumOperands() + 2612 OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) { 2613 const SCEV *Term = getZero(Ty); 2614 for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) { 2615 uint64_t Coeff1 = Choose(x, 2*x - y, Overflow); 2616 for (int z = std::max(y-x, y-(int)AddRec->getNumOperands()+1), 2617 ze = std::min(x+1, (int)OtherAddRec->getNumOperands()); 2618 z < ze && !Overflow; ++z) { 2619 uint64_t Coeff2 = Choose(2*x - y, x-z, Overflow); 2620 uint64_t Coeff; 2621 if (LargerThan64Bits) 2622 Coeff = umul_ov(Coeff1, Coeff2, Overflow); 2623 else 2624 Coeff = Coeff1*Coeff2; 2625 const SCEV *CoeffTerm = getConstant(Ty, Coeff); 2626 const SCEV *Term1 = AddRec->getOperand(y-z); 2627 const SCEV *Term2 = OtherAddRec->getOperand(z); 2628 Term = getAddExpr(Term, getMulExpr(CoeffTerm, Term1,Term2)); 2629 } 2630 } 2631 AddRecOps.push_back(Term); 2632 } 2633 if (!Overflow) { 2634 const SCEV *NewAddRec = getAddRecExpr(AddRecOps, AddRec->getLoop(), 2635 SCEV::FlagAnyWrap); 2636 if (Ops.size() == 2) return NewAddRec; 2637 Ops[Idx] = NewAddRec; 2638 Ops.erase(Ops.begin() + OtherIdx); --OtherIdx; 2639 OpsModified = true; 2640 AddRec = dyn_cast<SCEVAddRecExpr>(NewAddRec); 2641 if (!AddRec) 2642 break; 2643 } 2644 } 2645 if (OpsModified) 2646 return getMulExpr(Ops); 2647 2648 // Otherwise couldn't fold anything into this recurrence. Move onto the 2649 // next one. 2650 } 2651 2652 // Okay, it looks like we really DO need an mul expr. Check to see if we 2653 // already have one, otherwise create a new one. 2654 FoldingSetNodeID ID; 2655 ID.AddInteger(scMulExpr); 2656 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2657 ID.AddPointer(Ops[i]); 2658 void *IP = nullptr; 2659 SCEVMulExpr *S = 2660 static_cast<SCEVMulExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 2661 if (!S) { 2662 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 2663 std::uninitialized_copy(Ops.begin(), Ops.end(), O); 2664 S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator), 2665 O, Ops.size()); 2666 UniqueSCEVs.InsertNode(S, IP); 2667 } 2668 S->setNoWrapFlags(Flags); 2669 return S; 2670 } 2671 2672 /// Get a canonical unsigned division expression, or something simpler if 2673 /// possible. 2674 const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS, 2675 const SCEV *RHS) { 2676 assert(getEffectiveSCEVType(LHS->getType()) == 2677 getEffectiveSCEVType(RHS->getType()) && 2678 "SCEVUDivExpr operand types don't match!"); 2679 2680 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) { 2681 if (RHSC->getValue()->equalsInt(1)) 2682 return LHS; // X udiv 1 --> x 2683 // If the denominator is zero, the result of the udiv is undefined. Don't 2684 // try to analyze it, because the resolution chosen here may differ from 2685 // the resolution chosen in other parts of the compiler. 2686 if (!RHSC->getValue()->isZero()) { 2687 // Determine if the division can be folded into the operands of 2688 // its operands. 2689 // TODO: Generalize this to non-constants by using known-bits information. 2690 Type *Ty = LHS->getType(); 2691 unsigned LZ = RHSC->getAPInt().countLeadingZeros(); 2692 unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1; 2693 // For non-power-of-two values, effectively round the value up to the 2694 // nearest power of two. 2695 if (!RHSC->getAPInt().isPowerOf2()) 2696 ++MaxShiftAmt; 2697 IntegerType *ExtTy = 2698 IntegerType::get(getContext(), getTypeSizeInBits(Ty) + MaxShiftAmt); 2699 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS)) 2700 if (const SCEVConstant *Step = 2701 dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this))) { 2702 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded. 2703 const APInt &StepInt = Step->getAPInt(); 2704 const APInt &DivInt = RHSC->getAPInt(); 2705 if (!StepInt.urem(DivInt) && 2706 getZeroExtendExpr(AR, ExtTy) == 2707 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy), 2708 getZeroExtendExpr(Step, ExtTy), 2709 AR->getLoop(), SCEV::FlagAnyWrap)) { 2710 SmallVector<const SCEV *, 4> Operands; 2711 for (const SCEV *Op : AR->operands()) 2712 Operands.push_back(getUDivExpr(Op, RHS)); 2713 return getAddRecExpr(Operands, AR->getLoop(), SCEV::FlagNW); 2714 } 2715 /// Get a canonical UDivExpr for a recurrence. 2716 /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0. 2717 // We can currently only fold X%N if X is constant. 2718 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(AR->getStart()); 2719 if (StartC && !DivInt.urem(StepInt) && 2720 getZeroExtendExpr(AR, ExtTy) == 2721 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy), 2722 getZeroExtendExpr(Step, ExtTy), 2723 AR->getLoop(), SCEV::FlagAnyWrap)) { 2724 const APInt &StartInt = StartC->getAPInt(); 2725 const APInt &StartRem = StartInt.urem(StepInt); 2726 if (StartRem != 0) 2727 LHS = getAddRecExpr(getConstant(StartInt - StartRem), Step, 2728 AR->getLoop(), SCEV::FlagNW); 2729 } 2730 } 2731 // (A*B)/C --> A*(B/C) if safe and B/C can be folded. 2732 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) { 2733 SmallVector<const SCEV *, 4> Operands; 2734 for (const SCEV *Op : M->operands()) 2735 Operands.push_back(getZeroExtendExpr(Op, ExtTy)); 2736 if (getZeroExtendExpr(M, ExtTy) == getMulExpr(Operands)) 2737 // Find an operand that's safely divisible. 2738 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) { 2739 const SCEV *Op = M->getOperand(i); 2740 const SCEV *Div = getUDivExpr(Op, RHSC); 2741 if (!isa<SCEVUDivExpr>(Div) && getMulExpr(Div, RHSC) == Op) { 2742 Operands = SmallVector<const SCEV *, 4>(M->op_begin(), 2743 M->op_end()); 2744 Operands[i] = Div; 2745 return getMulExpr(Operands); 2746 } 2747 } 2748 } 2749 // (A+B)/C --> (A/C + B/C) if safe and A/C and B/C can be folded. 2750 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(LHS)) { 2751 SmallVector<const SCEV *, 4> Operands; 2752 for (const SCEV *Op : A->operands()) 2753 Operands.push_back(getZeroExtendExpr(Op, ExtTy)); 2754 if (getZeroExtendExpr(A, ExtTy) == getAddExpr(Operands)) { 2755 Operands.clear(); 2756 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) { 2757 const SCEV *Op = getUDivExpr(A->getOperand(i), RHS); 2758 if (isa<SCEVUDivExpr>(Op) || 2759 getMulExpr(Op, RHS) != A->getOperand(i)) 2760 break; 2761 Operands.push_back(Op); 2762 } 2763 if (Operands.size() == A->getNumOperands()) 2764 return getAddExpr(Operands); 2765 } 2766 } 2767 2768 // Fold if both operands are constant. 2769 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) { 2770 Constant *LHSCV = LHSC->getValue(); 2771 Constant *RHSCV = RHSC->getValue(); 2772 return getConstant(cast<ConstantInt>(ConstantExpr::getUDiv(LHSCV, 2773 RHSCV))); 2774 } 2775 } 2776 } 2777 2778 FoldingSetNodeID ID; 2779 ID.AddInteger(scUDivExpr); 2780 ID.AddPointer(LHS); 2781 ID.AddPointer(RHS); 2782 void *IP = nullptr; 2783 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 2784 SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator), 2785 LHS, RHS); 2786 UniqueSCEVs.InsertNode(S, IP); 2787 return S; 2788 } 2789 2790 static const APInt gcd(const SCEVConstant *C1, const SCEVConstant *C2) { 2791 APInt A = C1->getAPInt().abs(); 2792 APInt B = C2->getAPInt().abs(); 2793 uint32_t ABW = A.getBitWidth(); 2794 uint32_t BBW = B.getBitWidth(); 2795 2796 if (ABW > BBW) 2797 B = B.zext(ABW); 2798 else if (ABW < BBW) 2799 A = A.zext(BBW); 2800 2801 return APIntOps::GreatestCommonDivisor(A, B); 2802 } 2803 2804 /// Get a canonical unsigned division expression, or something simpler if 2805 /// possible. There is no representation for an exact udiv in SCEV IR, but we 2806 /// can attempt to remove factors from the LHS and RHS. We can't do this when 2807 /// it's not exact because the udiv may be clearing bits. 2808 const SCEV *ScalarEvolution::getUDivExactExpr(const SCEV *LHS, 2809 const SCEV *RHS) { 2810 // TODO: we could try to find factors in all sorts of things, but for now we 2811 // just deal with u/exact (multiply, constant). See SCEVDivision towards the 2812 // end of this file for inspiration. 2813 2814 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS); 2815 if (!Mul) 2816 return getUDivExpr(LHS, RHS); 2817 2818 if (const SCEVConstant *RHSCst = dyn_cast<SCEVConstant>(RHS)) { 2819 // If the mulexpr multiplies by a constant, then that constant must be the 2820 // first element of the mulexpr. 2821 if (const auto *LHSCst = dyn_cast<SCEVConstant>(Mul->getOperand(0))) { 2822 if (LHSCst == RHSCst) { 2823 SmallVector<const SCEV *, 2> Operands; 2824 Operands.append(Mul->op_begin() + 1, Mul->op_end()); 2825 return getMulExpr(Operands); 2826 } 2827 2828 // We can't just assume that LHSCst divides RHSCst cleanly, it could be 2829 // that there's a factor provided by one of the other terms. We need to 2830 // check. 2831 APInt Factor = gcd(LHSCst, RHSCst); 2832 if (!Factor.isIntN(1)) { 2833 LHSCst = 2834 cast<SCEVConstant>(getConstant(LHSCst->getAPInt().udiv(Factor))); 2835 RHSCst = 2836 cast<SCEVConstant>(getConstant(RHSCst->getAPInt().udiv(Factor))); 2837 SmallVector<const SCEV *, 2> Operands; 2838 Operands.push_back(LHSCst); 2839 Operands.append(Mul->op_begin() + 1, Mul->op_end()); 2840 LHS = getMulExpr(Operands); 2841 RHS = RHSCst; 2842 Mul = dyn_cast<SCEVMulExpr>(LHS); 2843 if (!Mul) 2844 return getUDivExactExpr(LHS, RHS); 2845 } 2846 } 2847 } 2848 2849 for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) { 2850 if (Mul->getOperand(i) == RHS) { 2851 SmallVector<const SCEV *, 2> Operands; 2852 Operands.append(Mul->op_begin(), Mul->op_begin() + i); 2853 Operands.append(Mul->op_begin() + i + 1, Mul->op_end()); 2854 return getMulExpr(Operands); 2855 } 2856 } 2857 2858 return getUDivExpr(LHS, RHS); 2859 } 2860 2861 /// Get an add recurrence expression for the specified loop. Simplify the 2862 /// expression as much as possible. 2863 const SCEV *ScalarEvolution::getAddRecExpr(const SCEV *Start, const SCEV *Step, 2864 const Loop *L, 2865 SCEV::NoWrapFlags Flags) { 2866 SmallVector<const SCEV *, 4> Operands; 2867 Operands.push_back(Start); 2868 if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Step)) 2869 if (StepChrec->getLoop() == L) { 2870 Operands.append(StepChrec->op_begin(), StepChrec->op_end()); 2871 return getAddRecExpr(Operands, L, maskFlags(Flags, SCEV::FlagNW)); 2872 } 2873 2874 Operands.push_back(Step); 2875 return getAddRecExpr(Operands, L, Flags); 2876 } 2877 2878 /// Get an add recurrence expression for the specified loop. Simplify the 2879 /// expression as much as possible. 2880 const SCEV * 2881 ScalarEvolution::getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands, 2882 const Loop *L, SCEV::NoWrapFlags Flags) { 2883 if (Operands.size() == 1) return Operands[0]; 2884 #ifndef NDEBUG 2885 Type *ETy = getEffectiveSCEVType(Operands[0]->getType()); 2886 for (unsigned i = 1, e = Operands.size(); i != e; ++i) 2887 assert(getEffectiveSCEVType(Operands[i]->getType()) == ETy && 2888 "SCEVAddRecExpr operand types don't match!"); 2889 for (unsigned i = 0, e = Operands.size(); i != e; ++i) 2890 assert(isLoopInvariant(Operands[i], L) && 2891 "SCEVAddRecExpr operand is not loop-invariant!"); 2892 #endif 2893 2894 if (Operands.back()->isZero()) { 2895 Operands.pop_back(); 2896 return getAddRecExpr(Operands, L, SCEV::FlagAnyWrap); // {X,+,0} --> X 2897 } 2898 2899 // It's tempting to want to call getMaxBackedgeTakenCount count here and 2900 // use that information to infer NUW and NSW flags. However, computing a 2901 // BE count requires calling getAddRecExpr, so we may not yet have a 2902 // meaningful BE count at this point (and if we don't, we'd be stuck 2903 // with a SCEVCouldNotCompute as the cached BE count). 2904 2905 Flags = StrengthenNoWrapFlags(this, scAddRecExpr, Operands, Flags); 2906 2907 // Canonicalize nested AddRecs in by nesting them in order of loop depth. 2908 if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Operands[0])) { 2909 const Loop *NestedLoop = NestedAR->getLoop(); 2910 if (L->contains(NestedLoop) 2911 ? (L->getLoopDepth() < NestedLoop->getLoopDepth()) 2912 : (!NestedLoop->contains(L) && 2913 DT.dominates(L->getHeader(), NestedLoop->getHeader()))) { 2914 SmallVector<const SCEV *, 4> NestedOperands(NestedAR->op_begin(), 2915 NestedAR->op_end()); 2916 Operands[0] = NestedAR->getStart(); 2917 // AddRecs require their operands be loop-invariant with respect to their 2918 // loops. Don't perform this transformation if it would break this 2919 // requirement. 2920 bool AllInvariant = all_of( 2921 Operands, [&](const SCEV *Op) { return isLoopInvariant(Op, L); }); 2922 2923 if (AllInvariant) { 2924 // Create a recurrence for the outer loop with the same step size. 2925 // 2926 // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the 2927 // inner recurrence has the same property. 2928 SCEV::NoWrapFlags OuterFlags = 2929 maskFlags(Flags, SCEV::FlagNW | NestedAR->getNoWrapFlags()); 2930 2931 NestedOperands[0] = getAddRecExpr(Operands, L, OuterFlags); 2932 AllInvariant = all_of(NestedOperands, [&](const SCEV *Op) { 2933 return isLoopInvariant(Op, NestedLoop); 2934 }); 2935 2936 if (AllInvariant) { 2937 // Ok, both add recurrences are valid after the transformation. 2938 // 2939 // The inner recurrence keeps its NW flag but only keeps NUW/NSW if 2940 // the outer recurrence has the same property. 2941 SCEV::NoWrapFlags InnerFlags = 2942 maskFlags(NestedAR->getNoWrapFlags(), SCEV::FlagNW | Flags); 2943 return getAddRecExpr(NestedOperands, NestedLoop, InnerFlags); 2944 } 2945 } 2946 // Reset Operands to its original state. 2947 Operands[0] = NestedAR; 2948 } 2949 } 2950 2951 // Okay, it looks like we really DO need an addrec expr. Check to see if we 2952 // already have one, otherwise create a new one. 2953 FoldingSetNodeID ID; 2954 ID.AddInteger(scAddRecExpr); 2955 for (unsigned i = 0, e = Operands.size(); i != e; ++i) 2956 ID.AddPointer(Operands[i]); 2957 ID.AddPointer(L); 2958 void *IP = nullptr; 2959 SCEVAddRecExpr *S = 2960 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 2961 if (!S) { 2962 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Operands.size()); 2963 std::uninitialized_copy(Operands.begin(), Operands.end(), O); 2964 S = new (SCEVAllocator) SCEVAddRecExpr(ID.Intern(SCEVAllocator), 2965 O, Operands.size(), L); 2966 UniqueSCEVs.InsertNode(S, IP); 2967 } 2968 S->setNoWrapFlags(Flags); 2969 return S; 2970 } 2971 2972 const SCEV * 2973 ScalarEvolution::getGEPExpr(Type *PointeeType, const SCEV *BaseExpr, 2974 const SmallVectorImpl<const SCEV *> &IndexExprs, 2975 bool InBounds) { 2976 // getSCEV(Base)->getType() has the same address space as Base->getType() 2977 // because SCEV::getType() preserves the address space. 2978 Type *IntPtrTy = getEffectiveSCEVType(BaseExpr->getType()); 2979 // FIXME(PR23527): Don't blindly transfer the inbounds flag from the GEP 2980 // instruction to its SCEV, because the Instruction may be guarded by control 2981 // flow and the no-overflow bits may not be valid for the expression in any 2982 // context. This can be fixed similarly to how these flags are handled for 2983 // adds. 2984 SCEV::NoWrapFlags Wrap = InBounds ? SCEV::FlagNSW : SCEV::FlagAnyWrap; 2985 2986 const SCEV *TotalOffset = getZero(IntPtrTy); 2987 // The address space is unimportant. The first thing we do on CurTy is getting 2988 // its element type. 2989 Type *CurTy = PointerType::getUnqual(PointeeType); 2990 for (const SCEV *IndexExpr : IndexExprs) { 2991 // Compute the (potentially symbolic) offset in bytes for this index. 2992 if (StructType *STy = dyn_cast<StructType>(CurTy)) { 2993 // For a struct, add the member offset. 2994 ConstantInt *Index = cast<SCEVConstant>(IndexExpr)->getValue(); 2995 unsigned FieldNo = Index->getZExtValue(); 2996 const SCEV *FieldOffset = getOffsetOfExpr(IntPtrTy, STy, FieldNo); 2997 2998 // Add the field offset to the running total offset. 2999 TotalOffset = getAddExpr(TotalOffset, FieldOffset); 3000 3001 // Update CurTy to the type of the field at Index. 3002 CurTy = STy->getTypeAtIndex(Index); 3003 } else { 3004 // Update CurTy to its element type. 3005 CurTy = cast<SequentialType>(CurTy)->getElementType(); 3006 // For an array, add the element offset, explicitly scaled. 3007 const SCEV *ElementSize = getSizeOfExpr(IntPtrTy, CurTy); 3008 // Getelementptr indices are signed. 3009 IndexExpr = getTruncateOrSignExtend(IndexExpr, IntPtrTy); 3010 3011 // Multiply the index by the element size to compute the element offset. 3012 const SCEV *LocalOffset = getMulExpr(IndexExpr, ElementSize, Wrap); 3013 3014 // Add the element offset to the running total offset. 3015 TotalOffset = getAddExpr(TotalOffset, LocalOffset); 3016 } 3017 } 3018 3019 // Add the total offset from all the GEP indices to the base. 3020 return getAddExpr(BaseExpr, TotalOffset, Wrap); 3021 } 3022 3023 const SCEV *ScalarEvolution::getSMaxExpr(const SCEV *LHS, 3024 const SCEV *RHS) { 3025 SmallVector<const SCEV *, 2> Ops = {LHS, RHS}; 3026 return getSMaxExpr(Ops); 3027 } 3028 3029 const SCEV * 3030 ScalarEvolution::getSMaxExpr(SmallVectorImpl<const SCEV *> &Ops) { 3031 assert(!Ops.empty() && "Cannot get empty smax!"); 3032 if (Ops.size() == 1) return Ops[0]; 3033 #ifndef NDEBUG 3034 Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 3035 for (unsigned i = 1, e = Ops.size(); i != e; ++i) 3036 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 3037 "SCEVSMaxExpr operand types don't match!"); 3038 #endif 3039 3040 // Sort by complexity, this groups all similar expression types together. 3041 GroupByComplexity(Ops, &LI); 3042 3043 // If there are any constants, fold them together. 3044 unsigned Idx = 0; 3045 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 3046 ++Idx; 3047 assert(Idx < Ops.size()); 3048 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 3049 // We found two constants, fold them together! 3050 ConstantInt *Fold = ConstantInt::get( 3051 getContext(), APIntOps::smax(LHSC->getAPInt(), RHSC->getAPInt())); 3052 Ops[0] = getConstant(Fold); 3053 Ops.erase(Ops.begin()+1); // Erase the folded element 3054 if (Ops.size() == 1) return Ops[0]; 3055 LHSC = cast<SCEVConstant>(Ops[0]); 3056 } 3057 3058 // If we are left with a constant minimum-int, strip it off. 3059 if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(true)) { 3060 Ops.erase(Ops.begin()); 3061 --Idx; 3062 } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(true)) { 3063 // If we have an smax with a constant maximum-int, it will always be 3064 // maximum-int. 3065 return Ops[0]; 3066 } 3067 3068 if (Ops.size() == 1) return Ops[0]; 3069 } 3070 3071 // Find the first SMax 3072 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scSMaxExpr) 3073 ++Idx; 3074 3075 // Check to see if one of the operands is an SMax. If so, expand its operands 3076 // onto our operand list, and recurse to simplify. 3077 if (Idx < Ops.size()) { 3078 bool DeletedSMax = false; 3079 while (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(Ops[Idx])) { 3080 Ops.erase(Ops.begin()+Idx); 3081 Ops.append(SMax->op_begin(), SMax->op_end()); 3082 DeletedSMax = true; 3083 } 3084 3085 if (DeletedSMax) 3086 return getSMaxExpr(Ops); 3087 } 3088 3089 // Okay, check to see if the same value occurs in the operand list twice. If 3090 // so, delete one. Since we sorted the list, these values are required to 3091 // be adjacent. 3092 for (unsigned i = 0, e = Ops.size()-1; i != e; ++i) 3093 // X smax Y smax Y --> X smax Y 3094 // X smax Y --> X, if X is always greater than Y 3095 if (Ops[i] == Ops[i+1] || 3096 isKnownPredicate(ICmpInst::ICMP_SGE, Ops[i], Ops[i+1])) { 3097 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2); 3098 --i; --e; 3099 } else if (isKnownPredicate(ICmpInst::ICMP_SLE, Ops[i], Ops[i+1])) { 3100 Ops.erase(Ops.begin()+i, Ops.begin()+i+1); 3101 --i; --e; 3102 } 3103 3104 if (Ops.size() == 1) return Ops[0]; 3105 3106 assert(!Ops.empty() && "Reduced smax down to nothing!"); 3107 3108 // Okay, it looks like we really DO need an smax expr. Check to see if we 3109 // already have one, otherwise create a new one. 3110 FoldingSetNodeID ID; 3111 ID.AddInteger(scSMaxExpr); 3112 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 3113 ID.AddPointer(Ops[i]); 3114 void *IP = nullptr; 3115 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 3116 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 3117 std::uninitialized_copy(Ops.begin(), Ops.end(), O); 3118 SCEV *S = new (SCEVAllocator) SCEVSMaxExpr(ID.Intern(SCEVAllocator), 3119 O, Ops.size()); 3120 UniqueSCEVs.InsertNode(S, IP); 3121 return S; 3122 } 3123 3124 const SCEV *ScalarEvolution::getUMaxExpr(const SCEV *LHS, 3125 const SCEV *RHS) { 3126 SmallVector<const SCEV *, 2> Ops = {LHS, RHS}; 3127 return getUMaxExpr(Ops); 3128 } 3129 3130 const SCEV * 3131 ScalarEvolution::getUMaxExpr(SmallVectorImpl<const SCEV *> &Ops) { 3132 assert(!Ops.empty() && "Cannot get empty umax!"); 3133 if (Ops.size() == 1) return Ops[0]; 3134 #ifndef NDEBUG 3135 Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 3136 for (unsigned i = 1, e = Ops.size(); i != e; ++i) 3137 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 3138 "SCEVUMaxExpr operand types don't match!"); 3139 #endif 3140 3141 // Sort by complexity, this groups all similar expression types together. 3142 GroupByComplexity(Ops, &LI); 3143 3144 // If there are any constants, fold them together. 3145 unsigned Idx = 0; 3146 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 3147 ++Idx; 3148 assert(Idx < Ops.size()); 3149 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 3150 // We found two constants, fold them together! 3151 ConstantInt *Fold = ConstantInt::get( 3152 getContext(), APIntOps::umax(LHSC->getAPInt(), RHSC->getAPInt())); 3153 Ops[0] = getConstant(Fold); 3154 Ops.erase(Ops.begin()+1); // Erase the folded element 3155 if (Ops.size() == 1) return Ops[0]; 3156 LHSC = cast<SCEVConstant>(Ops[0]); 3157 } 3158 3159 // If we are left with a constant minimum-int, strip it off. 3160 if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(false)) { 3161 Ops.erase(Ops.begin()); 3162 --Idx; 3163 } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(false)) { 3164 // If we have an umax with a constant maximum-int, it will always be 3165 // maximum-int. 3166 return Ops[0]; 3167 } 3168 3169 if (Ops.size() == 1) return Ops[0]; 3170 } 3171 3172 // Find the first UMax 3173 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scUMaxExpr) 3174 ++Idx; 3175 3176 // Check to see if one of the operands is a UMax. If so, expand its operands 3177 // onto our operand list, and recurse to simplify. 3178 if (Idx < Ops.size()) { 3179 bool DeletedUMax = false; 3180 while (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(Ops[Idx])) { 3181 Ops.erase(Ops.begin()+Idx); 3182 Ops.append(UMax->op_begin(), UMax->op_end()); 3183 DeletedUMax = true; 3184 } 3185 3186 if (DeletedUMax) 3187 return getUMaxExpr(Ops); 3188 } 3189 3190 // Okay, check to see if the same value occurs in the operand list twice. If 3191 // so, delete one. Since we sorted the list, these values are required to 3192 // be adjacent. 3193 for (unsigned i = 0, e = Ops.size()-1; i != e; ++i) 3194 // X umax Y umax Y --> X umax Y 3195 // X umax Y --> X, if X is always greater than Y 3196 if (Ops[i] == Ops[i+1] || 3197 isKnownPredicate(ICmpInst::ICMP_UGE, Ops[i], Ops[i+1])) { 3198 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2); 3199 --i; --e; 3200 } else if (isKnownPredicate(ICmpInst::ICMP_ULE, Ops[i], Ops[i+1])) { 3201 Ops.erase(Ops.begin()+i, Ops.begin()+i+1); 3202 --i; --e; 3203 } 3204 3205 if (Ops.size() == 1) return Ops[0]; 3206 3207 assert(!Ops.empty() && "Reduced umax down to nothing!"); 3208 3209 // Okay, it looks like we really DO need a umax expr. Check to see if we 3210 // already have one, otherwise create a new one. 3211 FoldingSetNodeID ID; 3212 ID.AddInteger(scUMaxExpr); 3213 for (unsigned i = 0, e = Ops.size(); i != e; ++i) 3214 ID.AddPointer(Ops[i]); 3215 void *IP = nullptr; 3216 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 3217 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 3218 std::uninitialized_copy(Ops.begin(), Ops.end(), O); 3219 SCEV *S = new (SCEVAllocator) SCEVUMaxExpr(ID.Intern(SCEVAllocator), 3220 O, Ops.size()); 3221 UniqueSCEVs.InsertNode(S, IP); 3222 return S; 3223 } 3224 3225 const SCEV *ScalarEvolution::getSMinExpr(const SCEV *LHS, 3226 const SCEV *RHS) { 3227 // ~smax(~x, ~y) == smin(x, y). 3228 return getNotSCEV(getSMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS))); 3229 } 3230 3231 const SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS, 3232 const SCEV *RHS) { 3233 // ~umax(~x, ~y) == umin(x, y) 3234 return getNotSCEV(getUMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS))); 3235 } 3236 3237 const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) { 3238 // We can bypass creating a target-independent 3239 // constant expression and then folding it back into a ConstantInt. 3240 // This is just a compile-time optimization. 3241 return getConstant(IntTy, getDataLayout().getTypeAllocSize(AllocTy)); 3242 } 3243 3244 const SCEV *ScalarEvolution::getOffsetOfExpr(Type *IntTy, 3245 StructType *STy, 3246 unsigned FieldNo) { 3247 // We can bypass creating a target-independent 3248 // constant expression and then folding it back into a ConstantInt. 3249 // This is just a compile-time optimization. 3250 return getConstant( 3251 IntTy, getDataLayout().getStructLayout(STy)->getElementOffset(FieldNo)); 3252 } 3253 3254 const SCEV *ScalarEvolution::getUnknown(Value *V) { 3255 // Don't attempt to do anything other than create a SCEVUnknown object 3256 // here. createSCEV only calls getUnknown after checking for all other 3257 // interesting possibilities, and any other code that calls getUnknown 3258 // is doing so in order to hide a value from SCEV canonicalization. 3259 3260 FoldingSetNodeID ID; 3261 ID.AddInteger(scUnknown); 3262 ID.AddPointer(V); 3263 void *IP = nullptr; 3264 if (SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) { 3265 assert(cast<SCEVUnknown>(S)->getValue() == V && 3266 "Stale SCEVUnknown in uniquing map!"); 3267 return S; 3268 } 3269 SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(SCEVAllocator), V, this, 3270 FirstUnknown); 3271 FirstUnknown = cast<SCEVUnknown>(S); 3272 UniqueSCEVs.InsertNode(S, IP); 3273 return S; 3274 } 3275 3276 //===----------------------------------------------------------------------===// 3277 // Basic SCEV Analysis and PHI Idiom Recognition Code 3278 // 3279 3280 /// Test if values of the given type are analyzable within the SCEV 3281 /// framework. This primarily includes integer types, and it can optionally 3282 /// include pointer types if the ScalarEvolution class has access to 3283 /// target-specific information. 3284 bool ScalarEvolution::isSCEVable(Type *Ty) const { 3285 // Integers and pointers are always SCEVable. 3286 return Ty->isIntegerTy() || Ty->isPointerTy(); 3287 } 3288 3289 /// Return the size in bits of the specified type, for which isSCEVable must 3290 /// return true. 3291 uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const { 3292 assert(isSCEVable(Ty) && "Type is not SCEVable!"); 3293 return getDataLayout().getTypeSizeInBits(Ty); 3294 } 3295 3296 /// Return a type with the same bitwidth as the given type and which represents 3297 /// how SCEV will treat the given type, for which isSCEVable must return 3298 /// true. For pointer types, this is the pointer-sized integer type. 3299 Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const { 3300 assert(isSCEVable(Ty) && "Type is not SCEVable!"); 3301 3302 if (Ty->isIntegerTy()) 3303 return Ty; 3304 3305 // The only other support type is pointer. 3306 assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!"); 3307 return getDataLayout().getIntPtrType(Ty); 3308 } 3309 3310 const SCEV *ScalarEvolution::getCouldNotCompute() { 3311 return CouldNotCompute.get(); 3312 } 3313 3314 3315 bool ScalarEvolution::checkValidity(const SCEV *S) const { 3316 // Helper class working with SCEVTraversal to figure out if a SCEV contains 3317 // a SCEVUnknown with null value-pointer. FindInvalidSCEVUnknown::FindOne 3318 // is set iff if find such SCEVUnknown. 3319 // 3320 struct FindInvalidSCEVUnknown { 3321 bool FindOne; 3322 FindInvalidSCEVUnknown() { FindOne = false; } 3323 bool follow(const SCEV *S) { 3324 switch (static_cast<SCEVTypes>(S->getSCEVType())) { 3325 case scConstant: 3326 return false; 3327 case scUnknown: 3328 if (!cast<SCEVUnknown>(S)->getValue()) 3329 FindOne = true; 3330 return false; 3331 default: 3332 return true; 3333 } 3334 } 3335 bool isDone() const { return FindOne; } 3336 }; 3337 3338 FindInvalidSCEVUnknown F; 3339 SCEVTraversal<FindInvalidSCEVUnknown> ST(F); 3340 ST.visitAll(S); 3341 3342 return !F.FindOne; 3343 } 3344 3345 namespace { 3346 // Helper class working with SCEVTraversal to figure out if a SCEV contains 3347 // a sub SCEV of scAddRecExpr type. FindInvalidSCEVUnknown::FoundOne is set 3348 // iff if such sub scAddRecExpr type SCEV is found. 3349 struct FindAddRecurrence { 3350 bool FoundOne; 3351 FindAddRecurrence() : FoundOne(false) {} 3352 3353 bool follow(const SCEV *S) { 3354 switch (static_cast<SCEVTypes>(S->getSCEVType())) { 3355 case scAddRecExpr: 3356 FoundOne = true; 3357 case scConstant: 3358 case scUnknown: 3359 case scCouldNotCompute: 3360 return false; 3361 default: 3362 return true; 3363 } 3364 } 3365 bool isDone() const { return FoundOne; } 3366 }; 3367 } 3368 3369 bool ScalarEvolution::containsAddRecurrence(const SCEV *S) { 3370 HasRecMapType::iterator I = HasRecMap.find_as(S); 3371 if (I != HasRecMap.end()) 3372 return I->second; 3373 3374 FindAddRecurrence F; 3375 SCEVTraversal<FindAddRecurrence> ST(F); 3376 ST.visitAll(S); 3377 HasRecMap.insert({S, F.FoundOne}); 3378 return F.FoundOne; 3379 } 3380 3381 /// Return the Value set from S. 3382 SetVector<Value *> *ScalarEvolution::getSCEVValues(const SCEV *S) { 3383 ExprValueMapType::iterator SI = ExprValueMap.find_as(S); 3384 if (SI == ExprValueMap.end()) 3385 return nullptr; 3386 #ifndef NDEBUG 3387 if (VerifySCEVMap) { 3388 // Check there is no dangling Value in the set returned. 3389 for (const auto &VE : SI->second) 3390 assert(ValueExprMap.count(VE)); 3391 } 3392 #endif 3393 return &SI->second; 3394 } 3395 3396 /// Erase Value from ValueExprMap and ExprValueMap. If ValueExprMap.erase(V) is 3397 /// not used together with forgetMemoizedResults(S), eraseValueFromMap should be 3398 /// used instead to ensure whenever V->S is removed from ValueExprMap, V is also 3399 /// removed from the set of ExprValueMap[S]. 3400 void ScalarEvolution::eraseValueFromMap(Value *V) { 3401 ValueExprMapType::iterator I = ValueExprMap.find_as(V); 3402 if (I != ValueExprMap.end()) { 3403 const SCEV *S = I->second; 3404 SetVector<Value *> *SV = getSCEVValues(S); 3405 // Remove V from the set of ExprValueMap[S] 3406 if (SV) 3407 SV->remove(V); 3408 ValueExprMap.erase(V); 3409 } 3410 } 3411 3412 /// Return an existing SCEV if it exists, otherwise analyze the expression and 3413 /// create a new one. 3414 const SCEV *ScalarEvolution::getSCEV(Value *V) { 3415 assert(isSCEVable(V->getType()) && "Value is not SCEVable!"); 3416 3417 const SCEV *S = getExistingSCEV(V); 3418 if (S == nullptr) { 3419 S = createSCEV(V); 3420 // During PHI resolution, it is possible to create two SCEVs for the same 3421 // V, so it is needed to double check whether V->S is inserted into 3422 // ValueExprMap before insert S->V into ExprValueMap. 3423 std::pair<ValueExprMapType::iterator, bool> Pair = 3424 ValueExprMap.insert({SCEVCallbackVH(V, this), S}); 3425 if (Pair.second) 3426 ExprValueMap[S].insert(V); 3427 } 3428 return S; 3429 } 3430 3431 const SCEV *ScalarEvolution::getExistingSCEV(Value *V) { 3432 assert(isSCEVable(V->getType()) && "Value is not SCEVable!"); 3433 3434 ValueExprMapType::iterator I = ValueExprMap.find_as(V); 3435 if (I != ValueExprMap.end()) { 3436 const SCEV *S = I->second; 3437 if (checkValidity(S)) 3438 return S; 3439 forgetMemoizedResults(S); 3440 ValueExprMap.erase(I); 3441 } 3442 return nullptr; 3443 } 3444 3445 /// Return a SCEV corresponding to -V = -1*V 3446 /// 3447 const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V, 3448 SCEV::NoWrapFlags Flags) { 3449 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V)) 3450 return getConstant( 3451 cast<ConstantInt>(ConstantExpr::getNeg(VC->getValue()))); 3452 3453 Type *Ty = V->getType(); 3454 Ty = getEffectiveSCEVType(Ty); 3455 return getMulExpr( 3456 V, getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty))), Flags); 3457 } 3458 3459 /// Return a SCEV corresponding to ~V = -1-V 3460 const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) { 3461 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V)) 3462 return getConstant( 3463 cast<ConstantInt>(ConstantExpr::getNot(VC->getValue()))); 3464 3465 Type *Ty = V->getType(); 3466 Ty = getEffectiveSCEVType(Ty); 3467 const SCEV *AllOnes = 3468 getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty))); 3469 return getMinusSCEV(AllOnes, V); 3470 } 3471 3472 const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS, 3473 SCEV::NoWrapFlags Flags) { 3474 // Fast path: X - X --> 0. 3475 if (LHS == RHS) 3476 return getZero(LHS->getType()); 3477 3478 // We represent LHS - RHS as LHS + (-1)*RHS. This transformation 3479 // makes it so that we cannot make much use of NUW. 3480 auto AddFlags = SCEV::FlagAnyWrap; 3481 const bool RHSIsNotMinSigned = 3482 !getSignedRange(RHS).getSignedMin().isMinSignedValue(); 3483 if (maskFlags(Flags, SCEV::FlagNSW) == SCEV::FlagNSW) { 3484 // Let M be the minimum representable signed value. Then (-1)*RHS 3485 // signed-wraps if and only if RHS is M. That can happen even for 3486 // a NSW subtraction because e.g. (-1)*M signed-wraps even though 3487 // -1 - M does not. So to transfer NSW from LHS - RHS to LHS + 3488 // (-1)*RHS, we need to prove that RHS != M. 3489 // 3490 // If LHS is non-negative and we know that LHS - RHS does not 3491 // signed-wrap, then RHS cannot be M. So we can rule out signed-wrap 3492 // either by proving that RHS > M or that LHS >= 0. 3493 if (RHSIsNotMinSigned || isKnownNonNegative(LHS)) { 3494 AddFlags = SCEV::FlagNSW; 3495 } 3496 } 3497 3498 // FIXME: Find a correct way to transfer NSW to (-1)*M when LHS - 3499 // RHS is NSW and LHS >= 0. 3500 // 3501 // The difficulty here is that the NSW flag may have been proven 3502 // relative to a loop that is to be found in a recurrence in LHS and 3503 // not in RHS. Applying NSW to (-1)*M may then let the NSW have a 3504 // larger scope than intended. 3505 auto NegFlags = RHSIsNotMinSigned ? SCEV::FlagNSW : SCEV::FlagAnyWrap; 3506 3507 return getAddExpr(LHS, getNegativeSCEV(RHS, NegFlags), AddFlags); 3508 } 3509 3510 const SCEV * 3511 ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty) { 3512 Type *SrcTy = V->getType(); 3513 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3514 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3515 "Cannot truncate or zero extend with non-integer arguments!"); 3516 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3517 return V; // No conversion 3518 if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty)) 3519 return getTruncateExpr(V, Ty); 3520 return getZeroExtendExpr(V, Ty); 3521 } 3522 3523 const SCEV * 3524 ScalarEvolution::getTruncateOrSignExtend(const SCEV *V, 3525 Type *Ty) { 3526 Type *SrcTy = V->getType(); 3527 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3528 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3529 "Cannot truncate or zero extend with non-integer arguments!"); 3530 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3531 return V; // No conversion 3532 if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty)) 3533 return getTruncateExpr(V, Ty); 3534 return getSignExtendExpr(V, Ty); 3535 } 3536 3537 const SCEV * 3538 ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) { 3539 Type *SrcTy = V->getType(); 3540 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3541 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3542 "Cannot noop or zero extend with non-integer arguments!"); 3543 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && 3544 "getNoopOrZeroExtend cannot truncate!"); 3545 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3546 return V; // No conversion 3547 return getZeroExtendExpr(V, Ty); 3548 } 3549 3550 const SCEV * 3551 ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) { 3552 Type *SrcTy = V->getType(); 3553 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3554 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3555 "Cannot noop or sign extend with non-integer arguments!"); 3556 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && 3557 "getNoopOrSignExtend cannot truncate!"); 3558 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3559 return V; // No conversion 3560 return getSignExtendExpr(V, Ty); 3561 } 3562 3563 const SCEV * 3564 ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) { 3565 Type *SrcTy = V->getType(); 3566 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3567 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3568 "Cannot noop or any extend with non-integer arguments!"); 3569 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && 3570 "getNoopOrAnyExtend cannot truncate!"); 3571 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3572 return V; // No conversion 3573 return getAnyExtendExpr(V, Ty); 3574 } 3575 3576 const SCEV * 3577 ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) { 3578 Type *SrcTy = V->getType(); 3579 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 3580 (Ty->isIntegerTy() || Ty->isPointerTy()) && 3581 "Cannot truncate or noop with non-integer arguments!"); 3582 assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) && 3583 "getTruncateOrNoop cannot extend!"); 3584 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 3585 return V; // No conversion 3586 return getTruncateExpr(V, Ty); 3587 } 3588 3589 const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS, 3590 const SCEV *RHS) { 3591 const SCEV *PromotedLHS = LHS; 3592 const SCEV *PromotedRHS = RHS; 3593 3594 if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType())) 3595 PromotedRHS = getZeroExtendExpr(RHS, LHS->getType()); 3596 else 3597 PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType()); 3598 3599 return getUMaxExpr(PromotedLHS, PromotedRHS); 3600 } 3601 3602 const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS, 3603 const SCEV *RHS) { 3604 const SCEV *PromotedLHS = LHS; 3605 const SCEV *PromotedRHS = RHS; 3606 3607 if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType())) 3608 PromotedRHS = getZeroExtendExpr(RHS, LHS->getType()); 3609 else 3610 PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType()); 3611 3612 return getUMinExpr(PromotedLHS, PromotedRHS); 3613 } 3614 3615 const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) { 3616 // A pointer operand may evaluate to a nonpointer expression, such as null. 3617 if (!V->getType()->isPointerTy()) 3618 return V; 3619 3620 if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(V)) { 3621 return getPointerBase(Cast->getOperand()); 3622 } else if (const SCEVNAryExpr *NAry = dyn_cast<SCEVNAryExpr>(V)) { 3623 const SCEV *PtrOp = nullptr; 3624 for (const SCEV *NAryOp : NAry->operands()) { 3625 if (NAryOp->getType()->isPointerTy()) { 3626 // Cannot find the base of an expression with multiple pointer operands. 3627 if (PtrOp) 3628 return V; 3629 PtrOp = NAryOp; 3630 } 3631 } 3632 if (!PtrOp) 3633 return V; 3634 return getPointerBase(PtrOp); 3635 } 3636 return V; 3637 } 3638 3639 /// Push users of the given Instruction onto the given Worklist. 3640 static void 3641 PushDefUseChildren(Instruction *I, 3642 SmallVectorImpl<Instruction *> &Worklist) { 3643 // Push the def-use children onto the Worklist stack. 3644 for (User *U : I->users()) 3645 Worklist.push_back(cast<Instruction>(U)); 3646 } 3647 3648 void ScalarEvolution::forgetSymbolicName(Instruction *PN, const SCEV *SymName) { 3649 SmallVector<Instruction *, 16> Worklist; 3650 PushDefUseChildren(PN, Worklist); 3651 3652 SmallPtrSet<Instruction *, 8> Visited; 3653 Visited.insert(PN); 3654 while (!Worklist.empty()) { 3655 Instruction *I = Worklist.pop_back_val(); 3656 if (!Visited.insert(I).second) 3657 continue; 3658 3659 auto It = ValueExprMap.find_as(static_cast<Value *>(I)); 3660 if (It != ValueExprMap.end()) { 3661 const SCEV *Old = It->second; 3662 3663 // Short-circuit the def-use traversal if the symbolic name 3664 // ceases to appear in expressions. 3665 if (Old != SymName && !hasOperand(Old, SymName)) 3666 continue; 3667 3668 // SCEVUnknown for a PHI either means that it has an unrecognized 3669 // structure, it's a PHI that's in the progress of being computed 3670 // by createNodeForPHI, or it's a single-value PHI. In the first case, 3671 // additional loop trip count information isn't going to change anything. 3672 // In the second case, createNodeForPHI will perform the necessary 3673 // updates on its own when it gets to that point. In the third, we do 3674 // want to forget the SCEVUnknown. 3675 if (!isa<PHINode>(I) || 3676 !isa<SCEVUnknown>(Old) || 3677 (I != PN && Old == SymName)) { 3678 forgetMemoizedResults(Old); 3679 ValueExprMap.erase(It); 3680 } 3681 } 3682 3683 PushDefUseChildren(I, Worklist); 3684 } 3685 } 3686 3687 namespace { 3688 class SCEVInitRewriter : public SCEVRewriteVisitor<SCEVInitRewriter> { 3689 public: 3690 static const SCEV *rewrite(const SCEV *S, const Loop *L, 3691 ScalarEvolution &SE) { 3692 SCEVInitRewriter Rewriter(L, SE); 3693 const SCEV *Result = Rewriter.visit(S); 3694 return Rewriter.isValid() ? Result : SE.getCouldNotCompute(); 3695 } 3696 3697 SCEVInitRewriter(const Loop *L, ScalarEvolution &SE) 3698 : SCEVRewriteVisitor(SE), L(L), Valid(true) {} 3699 3700 const SCEV *visitUnknown(const SCEVUnknown *Expr) { 3701 if (!(SE.getLoopDisposition(Expr, L) == ScalarEvolution::LoopInvariant)) 3702 Valid = false; 3703 return Expr; 3704 } 3705 3706 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { 3707 // Only allow AddRecExprs for this loop. 3708 if (Expr->getLoop() == L) 3709 return Expr->getStart(); 3710 Valid = false; 3711 return Expr; 3712 } 3713 3714 bool isValid() { return Valid; } 3715 3716 private: 3717 const Loop *L; 3718 bool Valid; 3719 }; 3720 3721 class SCEVShiftRewriter : public SCEVRewriteVisitor<SCEVShiftRewriter> { 3722 public: 3723 static const SCEV *rewrite(const SCEV *S, const Loop *L, 3724 ScalarEvolution &SE) { 3725 SCEVShiftRewriter Rewriter(L, SE); 3726 const SCEV *Result = Rewriter.visit(S); 3727 return Rewriter.isValid() ? Result : SE.getCouldNotCompute(); 3728 } 3729 3730 SCEVShiftRewriter(const Loop *L, ScalarEvolution &SE) 3731 : SCEVRewriteVisitor(SE), L(L), Valid(true) {} 3732 3733 const SCEV *visitUnknown(const SCEVUnknown *Expr) { 3734 // Only allow AddRecExprs for this loop. 3735 if (!(SE.getLoopDisposition(Expr, L) == ScalarEvolution::LoopInvariant)) 3736 Valid = false; 3737 return Expr; 3738 } 3739 3740 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { 3741 if (Expr->getLoop() == L && Expr->isAffine()) 3742 return SE.getMinusSCEV(Expr, Expr->getStepRecurrence(SE)); 3743 Valid = false; 3744 return Expr; 3745 } 3746 bool isValid() { return Valid; } 3747 3748 private: 3749 const Loop *L; 3750 bool Valid; 3751 }; 3752 } // end anonymous namespace 3753 3754 SCEV::NoWrapFlags 3755 ScalarEvolution::proveNoWrapViaConstantRanges(const SCEVAddRecExpr *AR) { 3756 if (!AR->isAffine()) 3757 return SCEV::FlagAnyWrap; 3758 3759 typedef OverflowingBinaryOperator OBO; 3760 SCEV::NoWrapFlags Result = SCEV::FlagAnyWrap; 3761 3762 if (!AR->hasNoSignedWrap()) { 3763 ConstantRange AddRecRange = getSignedRange(AR); 3764 ConstantRange IncRange = getSignedRange(AR->getStepRecurrence(*this)); 3765 3766 auto NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion( 3767 Instruction::Add, IncRange, OBO::NoSignedWrap); 3768 if (NSWRegion.contains(AddRecRange)) 3769 Result = ScalarEvolution::setFlags(Result, SCEV::FlagNSW); 3770 } 3771 3772 if (!AR->hasNoUnsignedWrap()) { 3773 ConstantRange AddRecRange = getUnsignedRange(AR); 3774 ConstantRange IncRange = getUnsignedRange(AR->getStepRecurrence(*this)); 3775 3776 auto NUWRegion = ConstantRange::makeGuaranteedNoWrapRegion( 3777 Instruction::Add, IncRange, OBO::NoUnsignedWrap); 3778 if (NUWRegion.contains(AddRecRange)) 3779 Result = ScalarEvolution::setFlags(Result, SCEV::FlagNUW); 3780 } 3781 3782 return Result; 3783 } 3784 3785 namespace { 3786 /// Represents an abstract binary operation. This may exist as a 3787 /// normal instruction or constant expression, or may have been 3788 /// derived from an expression tree. 3789 struct BinaryOp { 3790 unsigned Opcode; 3791 Value *LHS; 3792 Value *RHS; 3793 bool IsNSW; 3794 bool IsNUW; 3795 3796 /// Op is set if this BinaryOp corresponds to a concrete LLVM instruction or 3797 /// constant expression. 3798 Operator *Op; 3799 3800 explicit BinaryOp(Operator *Op) 3801 : Opcode(Op->getOpcode()), LHS(Op->getOperand(0)), RHS(Op->getOperand(1)), 3802 IsNSW(false), IsNUW(false), Op(Op) { 3803 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Op)) { 3804 IsNSW = OBO->hasNoSignedWrap(); 3805 IsNUW = OBO->hasNoUnsignedWrap(); 3806 } 3807 } 3808 3809 explicit BinaryOp(unsigned Opcode, Value *LHS, Value *RHS, bool IsNSW = false, 3810 bool IsNUW = false) 3811 : Opcode(Opcode), LHS(LHS), RHS(RHS), IsNSW(IsNSW), IsNUW(IsNUW), 3812 Op(nullptr) {} 3813 }; 3814 } 3815 3816 3817 /// Try to map \p V into a BinaryOp, and return \c None on failure. 3818 static Optional<BinaryOp> MatchBinaryOp(Value *V, DominatorTree &DT) { 3819 auto *Op = dyn_cast<Operator>(V); 3820 if (!Op) 3821 return None; 3822 3823 // Implementation detail: all the cleverness here should happen without 3824 // creating new SCEV expressions -- our caller knowns tricks to avoid creating 3825 // SCEV expressions when possible, and we should not break that. 3826 3827 switch (Op->getOpcode()) { 3828 case Instruction::Add: 3829 case Instruction::Sub: 3830 case Instruction::Mul: 3831 case Instruction::UDiv: 3832 case Instruction::And: 3833 case Instruction::Or: 3834 case Instruction::AShr: 3835 case Instruction::Shl: 3836 return BinaryOp(Op); 3837 3838 case Instruction::Xor: 3839 if (auto *RHSC = dyn_cast<ConstantInt>(Op->getOperand(1))) 3840 // If the RHS of the xor is a signbit, then this is just an add. 3841 // Instcombine turns add of signbit into xor as a strength reduction step. 3842 if (RHSC->getValue().isSignBit()) 3843 return BinaryOp(Instruction::Add, Op->getOperand(0), Op->getOperand(1)); 3844 return BinaryOp(Op); 3845 3846 case Instruction::LShr: 3847 // Turn logical shift right of a constant into a unsigned divide. 3848 if (ConstantInt *SA = dyn_cast<ConstantInt>(Op->getOperand(1))) { 3849 uint32_t BitWidth = cast<IntegerType>(Op->getType())->getBitWidth(); 3850 3851 // If the shift count is not less than the bitwidth, the result of 3852 // the shift is undefined. Don't try to analyze it, because the 3853 // resolution chosen here may differ from the resolution chosen in 3854 // other parts of the compiler. 3855 if (SA->getValue().ult(BitWidth)) { 3856 Constant *X = 3857 ConstantInt::get(SA->getContext(), 3858 APInt::getOneBitSet(BitWidth, SA->getZExtValue())); 3859 return BinaryOp(Instruction::UDiv, Op->getOperand(0), X); 3860 } 3861 } 3862 return BinaryOp(Op); 3863 3864 case Instruction::ExtractValue: { 3865 auto *EVI = cast<ExtractValueInst>(Op); 3866 if (EVI->getNumIndices() != 1 || EVI->getIndices()[0] != 0) 3867 break; 3868 3869 auto *CI = dyn_cast<CallInst>(EVI->getAggregateOperand()); 3870 if (!CI) 3871 break; 3872 3873 if (auto *F = CI->getCalledFunction()) 3874 switch (F->getIntrinsicID()) { 3875 case Intrinsic::sadd_with_overflow: 3876 case Intrinsic::uadd_with_overflow: { 3877 if (!isOverflowIntrinsicNoWrap(cast<IntrinsicInst>(CI), DT)) 3878 return BinaryOp(Instruction::Add, CI->getArgOperand(0), 3879 CI->getArgOperand(1)); 3880 3881 // Now that we know that all uses of the arithmetic-result component of 3882 // CI are guarded by the overflow check, we can go ahead and pretend 3883 // that the arithmetic is non-overflowing. 3884 if (F->getIntrinsicID() == Intrinsic::sadd_with_overflow) 3885 return BinaryOp(Instruction::Add, CI->getArgOperand(0), 3886 CI->getArgOperand(1), /* IsNSW = */ true, 3887 /* IsNUW = */ false); 3888 else 3889 return BinaryOp(Instruction::Add, CI->getArgOperand(0), 3890 CI->getArgOperand(1), /* IsNSW = */ false, 3891 /* IsNUW*/ true); 3892 } 3893 3894 case Intrinsic::ssub_with_overflow: 3895 case Intrinsic::usub_with_overflow: 3896 return BinaryOp(Instruction::Sub, CI->getArgOperand(0), 3897 CI->getArgOperand(1)); 3898 3899 case Intrinsic::smul_with_overflow: 3900 case Intrinsic::umul_with_overflow: 3901 return BinaryOp(Instruction::Mul, CI->getArgOperand(0), 3902 CI->getArgOperand(1)); 3903 default: 3904 break; 3905 } 3906 } 3907 3908 default: 3909 break; 3910 } 3911 3912 return None; 3913 } 3914 3915 const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) { 3916 const Loop *L = LI.getLoopFor(PN->getParent()); 3917 if (!L || L->getHeader() != PN->getParent()) 3918 return nullptr; 3919 3920 // The loop may have multiple entrances or multiple exits; we can analyze 3921 // this phi as an addrec if it has a unique entry value and a unique 3922 // backedge value. 3923 Value *BEValueV = nullptr, *StartValueV = nullptr; 3924 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { 3925 Value *V = PN->getIncomingValue(i); 3926 if (L->contains(PN->getIncomingBlock(i))) { 3927 if (!BEValueV) { 3928 BEValueV = V; 3929 } else if (BEValueV != V) { 3930 BEValueV = nullptr; 3931 break; 3932 } 3933 } else if (!StartValueV) { 3934 StartValueV = V; 3935 } else if (StartValueV != V) { 3936 StartValueV = nullptr; 3937 break; 3938 } 3939 } 3940 if (BEValueV && StartValueV) { 3941 // While we are analyzing this PHI node, handle its value symbolically. 3942 const SCEV *SymbolicName = getUnknown(PN); 3943 assert(ValueExprMap.find_as(PN) == ValueExprMap.end() && 3944 "PHI node already processed?"); 3945 ValueExprMap.insert({SCEVCallbackVH(PN, this), SymbolicName}); 3946 3947 // Using this symbolic name for the PHI, analyze the value coming around 3948 // the back-edge. 3949 const SCEV *BEValue = getSCEV(BEValueV); 3950 3951 // NOTE: If BEValue is loop invariant, we know that the PHI node just 3952 // has a special value for the first iteration of the loop. 3953 3954 // If the value coming around the backedge is an add with the symbolic 3955 // value we just inserted, then we found a simple induction variable! 3956 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(BEValue)) { 3957 // If there is a single occurrence of the symbolic value, replace it 3958 // with a recurrence. 3959 unsigned FoundIndex = Add->getNumOperands(); 3960 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i) 3961 if (Add->getOperand(i) == SymbolicName) 3962 if (FoundIndex == e) { 3963 FoundIndex = i; 3964 break; 3965 } 3966 3967 if (FoundIndex != Add->getNumOperands()) { 3968 // Create an add with everything but the specified operand. 3969 SmallVector<const SCEV *, 8> Ops; 3970 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i) 3971 if (i != FoundIndex) 3972 Ops.push_back(Add->getOperand(i)); 3973 const SCEV *Accum = getAddExpr(Ops); 3974 3975 // This is not a valid addrec if the step amount is varying each 3976 // loop iteration, but is not itself an addrec in this loop. 3977 if (isLoopInvariant(Accum, L) || 3978 (isa<SCEVAddRecExpr>(Accum) && 3979 cast<SCEVAddRecExpr>(Accum)->getLoop() == L)) { 3980 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap; 3981 3982 if (auto BO = MatchBinaryOp(BEValueV, DT)) { 3983 if (BO->Opcode == Instruction::Add && BO->LHS == PN) { 3984 if (BO->IsNUW) 3985 Flags = setFlags(Flags, SCEV::FlagNUW); 3986 if (BO->IsNSW) 3987 Flags = setFlags(Flags, SCEV::FlagNSW); 3988 } 3989 } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(BEValueV)) { 3990 // If the increment is an inbounds GEP, then we know the address 3991 // space cannot be wrapped around. We cannot make any guarantee 3992 // about signed or unsigned overflow because pointers are 3993 // unsigned but we may have a negative index from the base 3994 // pointer. We can guarantee that no unsigned wrap occurs if the 3995 // indices form a positive value. 3996 if (GEP->isInBounds() && GEP->getOperand(0) == PN) { 3997 Flags = setFlags(Flags, SCEV::FlagNW); 3998 3999 const SCEV *Ptr = getSCEV(GEP->getPointerOperand()); 4000 if (isKnownPositive(getMinusSCEV(getSCEV(GEP), Ptr))) 4001 Flags = setFlags(Flags, SCEV::FlagNUW); 4002 } 4003 4004 // We cannot transfer nuw and nsw flags from subtraction 4005 // operations -- sub nuw X, Y is not the same as add nuw X, -Y 4006 // for instance. 4007 } 4008 4009 const SCEV *StartVal = getSCEV(StartValueV); 4010 const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags); 4011 4012 // Okay, for the entire analysis of this edge we assumed the PHI 4013 // to be symbolic. We now need to go back and purge all of the 4014 // entries for the scalars that use the symbolic expression. 4015 forgetSymbolicName(PN, SymbolicName); 4016 ValueExprMap[SCEVCallbackVH(PN, this)] = PHISCEV; 4017 4018 // We can add Flags to the post-inc expression only if we 4019 // know that it us *undefined behavior* for BEValueV to 4020 // overflow. 4021 if (auto *BEInst = dyn_cast<Instruction>(BEValueV)) 4022 if (isLoopInvariant(Accum, L) && isAddRecNeverPoison(BEInst, L)) 4023 (void)getAddRecExpr(getAddExpr(StartVal, Accum), Accum, L, Flags); 4024 4025 return PHISCEV; 4026 } 4027 } 4028 } else { 4029 // Otherwise, this could be a loop like this: 4030 // i = 0; for (j = 1; ..; ++j) { .... i = j; } 4031 // In this case, j = {1,+,1} and BEValue is j. 4032 // Because the other in-value of i (0) fits the evolution of BEValue 4033 // i really is an addrec evolution. 4034 // 4035 // We can generalize this saying that i is the shifted value of BEValue 4036 // by one iteration: 4037 // PHI(f(0), f({1,+,1})) --> f({0,+,1}) 4038 const SCEV *Shifted = SCEVShiftRewriter::rewrite(BEValue, L, *this); 4039 const SCEV *Start = SCEVInitRewriter::rewrite(Shifted, L, *this); 4040 if (Shifted != getCouldNotCompute() && 4041 Start != getCouldNotCompute()) { 4042 const SCEV *StartVal = getSCEV(StartValueV); 4043 if (Start == StartVal) { 4044 // Okay, for the entire analysis of this edge we assumed the PHI 4045 // to be symbolic. We now need to go back and purge all of the 4046 // entries for the scalars that use the symbolic expression. 4047 forgetSymbolicName(PN, SymbolicName); 4048 ValueExprMap[SCEVCallbackVH(PN, this)] = Shifted; 4049 return Shifted; 4050 } 4051 } 4052 } 4053 4054 // Remove the temporary PHI node SCEV that has been inserted while intending 4055 // to create an AddRecExpr for this PHI node. We can not keep this temporary 4056 // as it will prevent later (possibly simpler) SCEV expressions to be added 4057 // to the ValueExprMap. 4058 ValueExprMap.erase(PN); 4059 } 4060 4061 return nullptr; 4062 } 4063 4064 // Checks if the SCEV S is available at BB. S is considered available at BB 4065 // if S can be materialized at BB without introducing a fault. 4066 static bool IsAvailableOnEntry(const Loop *L, DominatorTree &DT, const SCEV *S, 4067 BasicBlock *BB) { 4068 struct CheckAvailable { 4069 bool TraversalDone = false; 4070 bool Available = true; 4071 4072 const Loop *L = nullptr; // The loop BB is in (can be nullptr) 4073 BasicBlock *BB = nullptr; 4074 DominatorTree &DT; 4075 4076 CheckAvailable(const Loop *L, BasicBlock *BB, DominatorTree &DT) 4077 : L(L), BB(BB), DT(DT) {} 4078 4079 bool setUnavailable() { 4080 TraversalDone = true; 4081 Available = false; 4082 return false; 4083 } 4084 4085 bool follow(const SCEV *S) { 4086 switch (S->getSCEVType()) { 4087 case scConstant: case scTruncate: case scZeroExtend: case scSignExtend: 4088 case scAddExpr: case scMulExpr: case scUMaxExpr: case scSMaxExpr: 4089 // These expressions are available if their operand(s) is/are. 4090 return true; 4091 4092 case scAddRecExpr: { 4093 // We allow add recurrences that are on the loop BB is in, or some 4094 // outer loop. This guarantees availability because the value of the 4095 // add recurrence at BB is simply the "current" value of the induction 4096 // variable. We can relax this in the future; for instance an add 4097 // recurrence on a sibling dominating loop is also available at BB. 4098 const auto *ARLoop = cast<SCEVAddRecExpr>(S)->getLoop(); 4099 if (L && (ARLoop == L || ARLoop->contains(L))) 4100 return true; 4101 4102 return setUnavailable(); 4103 } 4104 4105 case scUnknown: { 4106 // For SCEVUnknown, we check for simple dominance. 4107 const auto *SU = cast<SCEVUnknown>(S); 4108 Value *V = SU->getValue(); 4109 4110 if (isa<Argument>(V)) 4111 return false; 4112 4113 if (isa<Instruction>(V) && DT.dominates(cast<Instruction>(V), BB)) 4114 return false; 4115 4116 return setUnavailable(); 4117 } 4118 4119 case scUDivExpr: 4120 case scCouldNotCompute: 4121 // We do not try to smart about these at all. 4122 return setUnavailable(); 4123 } 4124 llvm_unreachable("switch should be fully covered!"); 4125 } 4126 4127 bool isDone() { return TraversalDone; } 4128 }; 4129 4130 CheckAvailable CA(L, BB, DT); 4131 SCEVTraversal<CheckAvailable> ST(CA); 4132 4133 ST.visitAll(S); 4134 return CA.Available; 4135 } 4136 4137 // Try to match a control flow sequence that branches out at BI and merges back 4138 // at Merge into a "C ? LHS : RHS" select pattern. Return true on a successful 4139 // match. 4140 static bool BrPHIToSelect(DominatorTree &DT, BranchInst *BI, PHINode *Merge, 4141 Value *&C, Value *&LHS, Value *&RHS) { 4142 C = BI->getCondition(); 4143 4144 BasicBlockEdge LeftEdge(BI->getParent(), BI->getSuccessor(0)); 4145 BasicBlockEdge RightEdge(BI->getParent(), BI->getSuccessor(1)); 4146 4147 if (!LeftEdge.isSingleEdge()) 4148 return false; 4149 4150 assert(RightEdge.isSingleEdge() && "Follows from LeftEdge.isSingleEdge()"); 4151 4152 Use &LeftUse = Merge->getOperandUse(0); 4153 Use &RightUse = Merge->getOperandUse(1); 4154 4155 if (DT.dominates(LeftEdge, LeftUse) && DT.dominates(RightEdge, RightUse)) { 4156 LHS = LeftUse; 4157 RHS = RightUse; 4158 return true; 4159 } 4160 4161 if (DT.dominates(LeftEdge, RightUse) && DT.dominates(RightEdge, LeftUse)) { 4162 LHS = RightUse; 4163 RHS = LeftUse; 4164 return true; 4165 } 4166 4167 return false; 4168 } 4169 4170 const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) { 4171 if (PN->getNumIncomingValues() == 2) { 4172 const Loop *L = LI.getLoopFor(PN->getParent()); 4173 4174 // We don't want to break LCSSA, even in a SCEV expression tree. 4175 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) 4176 if (LI.getLoopFor(PN->getIncomingBlock(i)) != L) 4177 return nullptr; 4178 4179 // Try to match 4180 // 4181 // br %cond, label %left, label %right 4182 // left: 4183 // br label %merge 4184 // right: 4185 // br label %merge 4186 // merge: 4187 // V = phi [ %x, %left ], [ %y, %right ] 4188 // 4189 // as "select %cond, %x, %y" 4190 4191 BasicBlock *IDom = DT[PN->getParent()]->getIDom()->getBlock(); 4192 assert(IDom && "At least the entry block should dominate PN"); 4193 4194 auto *BI = dyn_cast<BranchInst>(IDom->getTerminator()); 4195 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr; 4196 4197 if (BI && BI->isConditional() && 4198 BrPHIToSelect(DT, BI, PN, Cond, LHS, RHS) && 4199 IsAvailableOnEntry(L, DT, getSCEV(LHS), PN->getParent()) && 4200 IsAvailableOnEntry(L, DT, getSCEV(RHS), PN->getParent())) 4201 return createNodeForSelectOrPHI(PN, Cond, LHS, RHS); 4202 } 4203 4204 return nullptr; 4205 } 4206 4207 const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) { 4208 if (const SCEV *S = createAddRecFromPHI(PN)) 4209 return S; 4210 4211 if (const SCEV *S = createNodeFromSelectLikePHI(PN)) 4212 return S; 4213 4214 // If the PHI has a single incoming value, follow that value, unless the 4215 // PHI's incoming blocks are in a different loop, in which case doing so 4216 // risks breaking LCSSA form. Instcombine would normally zap these, but 4217 // it doesn't have DominatorTree information, so it may miss cases. 4218 if (Value *V = SimplifyInstruction(PN, getDataLayout(), &TLI, &DT, &AC)) 4219 if (LI.replacementPreservesLCSSAForm(PN, V)) 4220 return getSCEV(V); 4221 4222 // If it's not a loop phi, we can't handle it yet. 4223 return getUnknown(PN); 4224 } 4225 4226 const SCEV *ScalarEvolution::createNodeForSelectOrPHI(Instruction *I, 4227 Value *Cond, 4228 Value *TrueVal, 4229 Value *FalseVal) { 4230 // Handle "constant" branch or select. This can occur for instance when a 4231 // loop pass transforms an inner loop and moves on to process the outer loop. 4232 if (auto *CI = dyn_cast<ConstantInt>(Cond)) 4233 return getSCEV(CI->isOne() ? TrueVal : FalseVal); 4234 4235 // Try to match some simple smax or umax patterns. 4236 auto *ICI = dyn_cast<ICmpInst>(Cond); 4237 if (!ICI) 4238 return getUnknown(I); 4239 4240 Value *LHS = ICI->getOperand(0); 4241 Value *RHS = ICI->getOperand(1); 4242 4243 switch (ICI->getPredicate()) { 4244 case ICmpInst::ICMP_SLT: 4245 case ICmpInst::ICMP_SLE: 4246 std::swap(LHS, RHS); 4247 // fall through 4248 case ICmpInst::ICMP_SGT: 4249 case ICmpInst::ICMP_SGE: 4250 // a >s b ? a+x : b+x -> smax(a, b)+x 4251 // a >s b ? b+x : a+x -> smin(a, b)+x 4252 if (getTypeSizeInBits(LHS->getType()) <= getTypeSizeInBits(I->getType())) { 4253 const SCEV *LS = getNoopOrSignExtend(getSCEV(LHS), I->getType()); 4254 const SCEV *RS = getNoopOrSignExtend(getSCEV(RHS), I->getType()); 4255 const SCEV *LA = getSCEV(TrueVal); 4256 const SCEV *RA = getSCEV(FalseVal); 4257 const SCEV *LDiff = getMinusSCEV(LA, LS); 4258 const SCEV *RDiff = getMinusSCEV(RA, RS); 4259 if (LDiff == RDiff) 4260 return getAddExpr(getSMaxExpr(LS, RS), LDiff); 4261 LDiff = getMinusSCEV(LA, RS); 4262 RDiff = getMinusSCEV(RA, LS); 4263 if (LDiff == RDiff) 4264 return getAddExpr(getSMinExpr(LS, RS), LDiff); 4265 } 4266 break; 4267 case ICmpInst::ICMP_ULT: 4268 case ICmpInst::ICMP_ULE: 4269 std::swap(LHS, RHS); 4270 // fall through 4271 case ICmpInst::ICMP_UGT: 4272 case ICmpInst::ICMP_UGE: 4273 // a >u b ? a+x : b+x -> umax(a, b)+x 4274 // a >u b ? b+x : a+x -> umin(a, b)+x 4275 if (getTypeSizeInBits(LHS->getType()) <= getTypeSizeInBits(I->getType())) { 4276 const SCEV *LS = getNoopOrZeroExtend(getSCEV(LHS), I->getType()); 4277 const SCEV *RS = getNoopOrZeroExtend(getSCEV(RHS), I->getType()); 4278 const SCEV *LA = getSCEV(TrueVal); 4279 const SCEV *RA = getSCEV(FalseVal); 4280 const SCEV *LDiff = getMinusSCEV(LA, LS); 4281 const SCEV *RDiff = getMinusSCEV(RA, RS); 4282 if (LDiff == RDiff) 4283 return getAddExpr(getUMaxExpr(LS, RS), LDiff); 4284 LDiff = getMinusSCEV(LA, RS); 4285 RDiff = getMinusSCEV(RA, LS); 4286 if (LDiff == RDiff) 4287 return getAddExpr(getUMinExpr(LS, RS), LDiff); 4288 } 4289 break; 4290 case ICmpInst::ICMP_NE: 4291 // n != 0 ? n+x : 1+x -> umax(n, 1)+x 4292 if (getTypeSizeInBits(LHS->getType()) <= getTypeSizeInBits(I->getType()) && 4293 isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isZero()) { 4294 const SCEV *One = getOne(I->getType()); 4295 const SCEV *LS = getNoopOrZeroExtend(getSCEV(LHS), I->getType()); 4296 const SCEV *LA = getSCEV(TrueVal); 4297 const SCEV *RA = getSCEV(FalseVal); 4298 const SCEV *LDiff = getMinusSCEV(LA, LS); 4299 const SCEV *RDiff = getMinusSCEV(RA, One); 4300 if (LDiff == RDiff) 4301 return getAddExpr(getUMaxExpr(One, LS), LDiff); 4302 } 4303 break; 4304 case ICmpInst::ICMP_EQ: 4305 // n == 0 ? 1+x : n+x -> umax(n, 1)+x 4306 if (getTypeSizeInBits(LHS->getType()) <= getTypeSizeInBits(I->getType()) && 4307 isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isZero()) { 4308 const SCEV *One = getOne(I->getType()); 4309 const SCEV *LS = getNoopOrZeroExtend(getSCEV(LHS), I->getType()); 4310 const SCEV *LA = getSCEV(TrueVal); 4311 const SCEV *RA = getSCEV(FalseVal); 4312 const SCEV *LDiff = getMinusSCEV(LA, One); 4313 const SCEV *RDiff = getMinusSCEV(RA, LS); 4314 if (LDiff == RDiff) 4315 return getAddExpr(getUMaxExpr(One, LS), LDiff); 4316 } 4317 break; 4318 default: 4319 break; 4320 } 4321 4322 return getUnknown(I); 4323 } 4324 4325 /// Expand GEP instructions into add and multiply operations. This allows them 4326 /// to be analyzed by regular SCEV code. 4327 const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) { 4328 // Don't attempt to analyze GEPs over unsized objects. 4329 if (!GEP->getSourceElementType()->isSized()) 4330 return getUnknown(GEP); 4331 4332 SmallVector<const SCEV *, 4> IndexExprs; 4333 for (auto Index = GEP->idx_begin(); Index != GEP->idx_end(); ++Index) 4334 IndexExprs.push_back(getSCEV(*Index)); 4335 return getGEPExpr(GEP->getSourceElementType(), 4336 getSCEV(GEP->getPointerOperand()), 4337 IndexExprs, GEP->isInBounds()); 4338 } 4339 4340 uint32_t 4341 ScalarEvolution::GetMinTrailingZeros(const SCEV *S) { 4342 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) 4343 return C->getAPInt().countTrailingZeros(); 4344 4345 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(S)) 4346 return std::min(GetMinTrailingZeros(T->getOperand()), 4347 (uint32_t)getTypeSizeInBits(T->getType())); 4348 4349 if (const SCEVZeroExtendExpr *E = dyn_cast<SCEVZeroExtendExpr>(S)) { 4350 uint32_t OpRes = GetMinTrailingZeros(E->getOperand()); 4351 return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ? 4352 getTypeSizeInBits(E->getType()) : OpRes; 4353 } 4354 4355 if (const SCEVSignExtendExpr *E = dyn_cast<SCEVSignExtendExpr>(S)) { 4356 uint32_t OpRes = GetMinTrailingZeros(E->getOperand()); 4357 return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ? 4358 getTypeSizeInBits(E->getType()) : OpRes; 4359 } 4360 4361 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(S)) { 4362 // The result is the min of all operands results. 4363 uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0)); 4364 for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i) 4365 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i))); 4366 return MinOpRes; 4367 } 4368 4369 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(S)) { 4370 // The result is the sum of all operands results. 4371 uint32_t SumOpRes = GetMinTrailingZeros(M->getOperand(0)); 4372 uint32_t BitWidth = getTypeSizeInBits(M->getType()); 4373 for (unsigned i = 1, e = M->getNumOperands(); 4374 SumOpRes != BitWidth && i != e; ++i) 4375 SumOpRes = std::min(SumOpRes + GetMinTrailingZeros(M->getOperand(i)), 4376 BitWidth); 4377 return SumOpRes; 4378 } 4379 4380 if (const SCEVAddRecExpr *A = dyn_cast<SCEVAddRecExpr>(S)) { 4381 // The result is the min of all operands results. 4382 uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0)); 4383 for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i) 4384 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i))); 4385 return MinOpRes; 4386 } 4387 4388 if (const SCEVSMaxExpr *M = dyn_cast<SCEVSMaxExpr>(S)) { 4389 // The result is the min of all operands results. 4390 uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0)); 4391 for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i) 4392 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i))); 4393 return MinOpRes; 4394 } 4395 4396 if (const SCEVUMaxExpr *M = dyn_cast<SCEVUMaxExpr>(S)) { 4397 // The result is the min of all operands results. 4398 uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0)); 4399 for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i) 4400 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i))); 4401 return MinOpRes; 4402 } 4403 4404 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 4405 // For a SCEVUnknown, ask ValueTracking. 4406 unsigned BitWidth = getTypeSizeInBits(U->getType()); 4407 APInt Zeros(BitWidth, 0), Ones(BitWidth, 0); 4408 computeKnownBits(U->getValue(), Zeros, Ones, getDataLayout(), 0, &AC, 4409 nullptr, &DT); 4410 return Zeros.countTrailingOnes(); 4411 } 4412 4413 // SCEVUDivExpr 4414 return 0; 4415 } 4416 4417 /// Helper method to assign a range to V from metadata present in the IR. 4418 static Optional<ConstantRange> GetRangeFromMetadata(Value *V) { 4419 if (Instruction *I = dyn_cast<Instruction>(V)) 4420 if (MDNode *MD = I->getMetadata(LLVMContext::MD_range)) 4421 return getConstantRangeFromMetadata(*MD); 4422 4423 return None; 4424 } 4425 4426 /// Determine the range for a particular SCEV. If SignHint is 4427 /// HINT_RANGE_UNSIGNED (resp. HINT_RANGE_SIGNED) then getRange prefers ranges 4428 /// with a "cleaner" unsigned (resp. signed) representation. 4429 ConstantRange 4430 ScalarEvolution::getRange(const SCEV *S, 4431 ScalarEvolution::RangeSignHint SignHint) { 4432 DenseMap<const SCEV *, ConstantRange> &Cache = 4433 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges 4434 : SignedRanges; 4435 4436 // See if we've computed this range already. 4437 DenseMap<const SCEV *, ConstantRange>::iterator I = Cache.find(S); 4438 if (I != Cache.end()) 4439 return I->second; 4440 4441 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) 4442 return setRange(C, SignHint, ConstantRange(C->getAPInt())); 4443 4444 unsigned BitWidth = getTypeSizeInBits(S->getType()); 4445 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true); 4446 4447 // If the value has known zeros, the maximum value will have those known zeros 4448 // as well. 4449 uint32_t TZ = GetMinTrailingZeros(S); 4450 if (TZ != 0) { 4451 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) 4452 ConservativeResult = 4453 ConstantRange(APInt::getMinValue(BitWidth), 4454 APInt::getMaxValue(BitWidth).lshr(TZ).shl(TZ) + 1); 4455 else 4456 ConservativeResult = ConstantRange( 4457 APInt::getSignedMinValue(BitWidth), 4458 APInt::getSignedMaxValue(BitWidth).ashr(TZ).shl(TZ) + 1); 4459 } 4460 4461 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 4462 ConstantRange X = getRange(Add->getOperand(0), SignHint); 4463 for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i) 4464 X = X.add(getRange(Add->getOperand(i), SignHint)); 4465 return setRange(Add, SignHint, ConservativeResult.intersectWith(X)); 4466 } 4467 4468 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) { 4469 ConstantRange X = getRange(Mul->getOperand(0), SignHint); 4470 for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i) 4471 X = X.multiply(getRange(Mul->getOperand(i), SignHint)); 4472 return setRange(Mul, SignHint, ConservativeResult.intersectWith(X)); 4473 } 4474 4475 if (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(S)) { 4476 ConstantRange X = getRange(SMax->getOperand(0), SignHint); 4477 for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i) 4478 X = X.smax(getRange(SMax->getOperand(i), SignHint)); 4479 return setRange(SMax, SignHint, ConservativeResult.intersectWith(X)); 4480 } 4481 4482 if (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(S)) { 4483 ConstantRange X = getRange(UMax->getOperand(0), SignHint); 4484 for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i) 4485 X = X.umax(getRange(UMax->getOperand(i), SignHint)); 4486 return setRange(UMax, SignHint, ConservativeResult.intersectWith(X)); 4487 } 4488 4489 if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) { 4490 ConstantRange X = getRange(UDiv->getLHS(), SignHint); 4491 ConstantRange Y = getRange(UDiv->getRHS(), SignHint); 4492 return setRange(UDiv, SignHint, 4493 ConservativeResult.intersectWith(X.udiv(Y))); 4494 } 4495 4496 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S)) { 4497 ConstantRange X = getRange(ZExt->getOperand(), SignHint); 4498 return setRange(ZExt, SignHint, 4499 ConservativeResult.intersectWith(X.zeroExtend(BitWidth))); 4500 } 4501 4502 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S)) { 4503 ConstantRange X = getRange(SExt->getOperand(), SignHint); 4504 return setRange(SExt, SignHint, 4505 ConservativeResult.intersectWith(X.signExtend(BitWidth))); 4506 } 4507 4508 if (const SCEVTruncateExpr *Trunc = dyn_cast<SCEVTruncateExpr>(S)) { 4509 ConstantRange X = getRange(Trunc->getOperand(), SignHint); 4510 return setRange(Trunc, SignHint, 4511 ConservativeResult.intersectWith(X.truncate(BitWidth))); 4512 } 4513 4514 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) { 4515 // If there's no unsigned wrap, the value will never be less than its 4516 // initial value. 4517 if (AddRec->hasNoUnsignedWrap()) 4518 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(AddRec->getStart())) 4519 if (!C->getValue()->isZero()) 4520 ConservativeResult = ConservativeResult.intersectWith( 4521 ConstantRange(C->getAPInt(), APInt(BitWidth, 0))); 4522 4523 // If there's no signed wrap, and all the operands have the same sign or 4524 // zero, the value won't ever change sign. 4525 if (AddRec->hasNoSignedWrap()) { 4526 bool AllNonNeg = true; 4527 bool AllNonPos = true; 4528 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) { 4529 if (!isKnownNonNegative(AddRec->getOperand(i))) AllNonNeg = false; 4530 if (!isKnownNonPositive(AddRec->getOperand(i))) AllNonPos = false; 4531 } 4532 if (AllNonNeg) 4533 ConservativeResult = ConservativeResult.intersectWith( 4534 ConstantRange(APInt(BitWidth, 0), 4535 APInt::getSignedMinValue(BitWidth))); 4536 else if (AllNonPos) 4537 ConservativeResult = ConservativeResult.intersectWith( 4538 ConstantRange(APInt::getSignedMinValue(BitWidth), 4539 APInt(BitWidth, 1))); 4540 } 4541 4542 // TODO: non-affine addrec 4543 if (AddRec->isAffine()) { 4544 const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop()); 4545 if (!isa<SCEVCouldNotCompute>(MaxBECount) && 4546 getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) { 4547 auto RangeFromAffine = getRangeForAffineAR( 4548 AddRec->getStart(), AddRec->getStepRecurrence(*this), MaxBECount, 4549 BitWidth); 4550 if (!RangeFromAffine.isFullSet()) 4551 ConservativeResult = 4552 ConservativeResult.intersectWith(RangeFromAffine); 4553 4554 auto RangeFromFactoring = getRangeViaFactoring( 4555 AddRec->getStart(), AddRec->getStepRecurrence(*this), MaxBECount, 4556 BitWidth); 4557 if (!RangeFromFactoring.isFullSet()) 4558 ConservativeResult = 4559 ConservativeResult.intersectWith(RangeFromFactoring); 4560 } 4561 } 4562 4563 return setRange(AddRec, SignHint, ConservativeResult); 4564 } 4565 4566 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 4567 // Check if the IR explicitly contains !range metadata. 4568 Optional<ConstantRange> MDRange = GetRangeFromMetadata(U->getValue()); 4569 if (MDRange.hasValue()) 4570 ConservativeResult = ConservativeResult.intersectWith(MDRange.getValue()); 4571 4572 // Split here to avoid paying the compile-time cost of calling both 4573 // computeKnownBits and ComputeNumSignBits. This restriction can be lifted 4574 // if needed. 4575 const DataLayout &DL = getDataLayout(); 4576 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) { 4577 // For a SCEVUnknown, ask ValueTracking. 4578 APInt Zeros(BitWidth, 0), Ones(BitWidth, 0); 4579 computeKnownBits(U->getValue(), Zeros, Ones, DL, 0, &AC, nullptr, &DT); 4580 if (Ones != ~Zeros + 1) 4581 ConservativeResult = 4582 ConservativeResult.intersectWith(ConstantRange(Ones, ~Zeros + 1)); 4583 } else { 4584 assert(SignHint == ScalarEvolution::HINT_RANGE_SIGNED && 4585 "generalize as needed!"); 4586 unsigned NS = ComputeNumSignBits(U->getValue(), DL, 0, &AC, nullptr, &DT); 4587 if (NS > 1) 4588 ConservativeResult = ConservativeResult.intersectWith( 4589 ConstantRange(APInt::getSignedMinValue(BitWidth).ashr(NS - 1), 4590 APInt::getSignedMaxValue(BitWidth).ashr(NS - 1) + 1)); 4591 } 4592 4593 return setRange(U, SignHint, ConservativeResult); 4594 } 4595 4596 return setRange(S, SignHint, ConservativeResult); 4597 } 4598 4599 ConstantRange ScalarEvolution::getRangeForAffineAR(const SCEV *Start, 4600 const SCEV *Step, 4601 const SCEV *MaxBECount, 4602 unsigned BitWidth) { 4603 assert(!isa<SCEVCouldNotCompute>(MaxBECount) && 4604 getTypeSizeInBits(MaxBECount->getType()) <= BitWidth && 4605 "Precondition!"); 4606 4607 ConstantRange Result(BitWidth, /* isFullSet = */ true); 4608 4609 // Check for overflow. This must be done with ConstantRange arithmetic 4610 // because we could be called from within the ScalarEvolution overflow 4611 // checking code. 4612 4613 MaxBECount = getNoopOrZeroExtend(MaxBECount, Start->getType()); 4614 ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount); 4615 ConstantRange ZExtMaxBECountRange = 4616 MaxBECountRange.zextOrTrunc(BitWidth * 2 + 1); 4617 4618 ConstantRange StepSRange = getSignedRange(Step); 4619 ConstantRange SExtStepSRange = StepSRange.sextOrTrunc(BitWidth * 2 + 1); 4620 4621 ConstantRange StartURange = getUnsignedRange(Start); 4622 ConstantRange EndURange = 4623 StartURange.add(MaxBECountRange.multiply(StepSRange)); 4624 4625 // Check for unsigned overflow. 4626 ConstantRange ZExtStartURange = StartURange.zextOrTrunc(BitWidth * 2 + 1); 4627 ConstantRange ZExtEndURange = EndURange.zextOrTrunc(BitWidth * 2 + 1); 4628 if (ZExtStartURange.add(ZExtMaxBECountRange.multiply(SExtStepSRange)) == 4629 ZExtEndURange) { 4630 APInt Min = APIntOps::umin(StartURange.getUnsignedMin(), 4631 EndURange.getUnsignedMin()); 4632 APInt Max = APIntOps::umax(StartURange.getUnsignedMax(), 4633 EndURange.getUnsignedMax()); 4634 bool IsFullRange = Min.isMinValue() && Max.isMaxValue(); 4635 if (!IsFullRange) 4636 Result = 4637 Result.intersectWith(ConstantRange(Min, Max + 1)); 4638 } 4639 4640 ConstantRange StartSRange = getSignedRange(Start); 4641 ConstantRange EndSRange = 4642 StartSRange.add(MaxBECountRange.multiply(StepSRange)); 4643 4644 // Check for signed overflow. This must be done with ConstantRange 4645 // arithmetic because we could be called from within the ScalarEvolution 4646 // overflow checking code. 4647 ConstantRange SExtStartSRange = StartSRange.sextOrTrunc(BitWidth * 2 + 1); 4648 ConstantRange SExtEndSRange = EndSRange.sextOrTrunc(BitWidth * 2 + 1); 4649 if (SExtStartSRange.add(ZExtMaxBECountRange.multiply(SExtStepSRange)) == 4650 SExtEndSRange) { 4651 APInt Min = 4652 APIntOps::smin(StartSRange.getSignedMin(), EndSRange.getSignedMin()); 4653 APInt Max = 4654 APIntOps::smax(StartSRange.getSignedMax(), EndSRange.getSignedMax()); 4655 bool IsFullRange = Min.isMinSignedValue() && Max.isMaxSignedValue(); 4656 if (!IsFullRange) 4657 Result = 4658 Result.intersectWith(ConstantRange(Min, Max + 1)); 4659 } 4660 4661 return Result; 4662 } 4663 4664 ConstantRange ScalarEvolution::getRangeViaFactoring(const SCEV *Start, 4665 const SCEV *Step, 4666 const SCEV *MaxBECount, 4667 unsigned BitWidth) { 4668 // RangeOf({C?A:B,+,C?P:Q}) == RangeOf(C?{A,+,P}:{B,+,Q}) 4669 // == RangeOf({A,+,P}) union RangeOf({B,+,Q}) 4670 4671 struct SelectPattern { 4672 Value *Condition = nullptr; 4673 APInt TrueValue; 4674 APInt FalseValue; 4675 4676 explicit SelectPattern(ScalarEvolution &SE, unsigned BitWidth, 4677 const SCEV *S) { 4678 Optional<unsigned> CastOp; 4679 APInt Offset(BitWidth, 0); 4680 4681 assert(SE.getTypeSizeInBits(S->getType()) == BitWidth && 4682 "Should be!"); 4683 4684 // Peel off a constant offset: 4685 if (auto *SA = dyn_cast<SCEVAddExpr>(S)) { 4686 // In the future we could consider being smarter here and handle 4687 // {Start+Step,+,Step} too. 4688 if (SA->getNumOperands() != 2 || !isa<SCEVConstant>(SA->getOperand(0))) 4689 return; 4690 4691 Offset = cast<SCEVConstant>(SA->getOperand(0))->getAPInt(); 4692 S = SA->getOperand(1); 4693 } 4694 4695 // Peel off a cast operation 4696 if (auto *SCast = dyn_cast<SCEVCastExpr>(S)) { 4697 CastOp = SCast->getSCEVType(); 4698 S = SCast->getOperand(); 4699 } 4700 4701 using namespace llvm::PatternMatch; 4702 4703 auto *SU = dyn_cast<SCEVUnknown>(S); 4704 const APInt *TrueVal, *FalseVal; 4705 if (!SU || 4706 !match(SU->getValue(), m_Select(m_Value(Condition), m_APInt(TrueVal), 4707 m_APInt(FalseVal)))) { 4708 Condition = nullptr; 4709 return; 4710 } 4711 4712 TrueValue = *TrueVal; 4713 FalseValue = *FalseVal; 4714 4715 // Re-apply the cast we peeled off earlier 4716 if (CastOp.hasValue()) 4717 switch (*CastOp) { 4718 default: 4719 llvm_unreachable("Unknown SCEV cast type!"); 4720 4721 case scTruncate: 4722 TrueValue = TrueValue.trunc(BitWidth); 4723 FalseValue = FalseValue.trunc(BitWidth); 4724 break; 4725 case scZeroExtend: 4726 TrueValue = TrueValue.zext(BitWidth); 4727 FalseValue = FalseValue.zext(BitWidth); 4728 break; 4729 case scSignExtend: 4730 TrueValue = TrueValue.sext(BitWidth); 4731 FalseValue = FalseValue.sext(BitWidth); 4732 break; 4733 } 4734 4735 // Re-apply the constant offset we peeled off earlier 4736 TrueValue += Offset; 4737 FalseValue += Offset; 4738 } 4739 4740 bool isRecognized() { return Condition != nullptr; } 4741 }; 4742 4743 SelectPattern StartPattern(*this, BitWidth, Start); 4744 if (!StartPattern.isRecognized()) 4745 return ConstantRange(BitWidth, /* isFullSet = */ true); 4746 4747 SelectPattern StepPattern(*this, BitWidth, Step); 4748 if (!StepPattern.isRecognized()) 4749 return ConstantRange(BitWidth, /* isFullSet = */ true); 4750 4751 if (StartPattern.Condition != StepPattern.Condition) { 4752 // We don't handle this case today; but we could, by considering four 4753 // possibilities below instead of two. I'm not sure if there are cases where 4754 // that will help over what getRange already does, though. 4755 return ConstantRange(BitWidth, /* isFullSet = */ true); 4756 } 4757 4758 // NB! Calling ScalarEvolution::getConstant is fine, but we should not try to 4759 // construct arbitrary general SCEV expressions here. This function is called 4760 // from deep in the call stack, and calling getSCEV (on a sext instruction, 4761 // say) can end up caching a suboptimal value. 4762 4763 // FIXME: without the explicit `this` receiver below, MSVC errors out with 4764 // C2352 and C2512 (otherwise it isn't needed). 4765 4766 const SCEV *TrueStart = this->getConstant(StartPattern.TrueValue); 4767 const SCEV *TrueStep = this->getConstant(StepPattern.TrueValue); 4768 const SCEV *FalseStart = this->getConstant(StartPattern.FalseValue); 4769 const SCEV *FalseStep = this->getConstant(StepPattern.FalseValue); 4770 4771 ConstantRange TrueRange = 4772 this->getRangeForAffineAR(TrueStart, TrueStep, MaxBECount, BitWidth); 4773 ConstantRange FalseRange = 4774 this->getRangeForAffineAR(FalseStart, FalseStep, MaxBECount, BitWidth); 4775 4776 return TrueRange.unionWith(FalseRange); 4777 } 4778 4779 SCEV::NoWrapFlags ScalarEvolution::getNoWrapFlagsFromUB(const Value *V) { 4780 if (isa<ConstantExpr>(V)) return SCEV::FlagAnyWrap; 4781 const BinaryOperator *BinOp = cast<BinaryOperator>(V); 4782 4783 // Return early if there are no flags to propagate to the SCEV. 4784 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap; 4785 if (BinOp->hasNoUnsignedWrap()) 4786 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW); 4787 if (BinOp->hasNoSignedWrap()) 4788 Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNSW); 4789 if (Flags == SCEV::FlagAnyWrap) 4790 return SCEV::FlagAnyWrap; 4791 4792 return isSCEVExprNeverPoison(BinOp) ? Flags : SCEV::FlagAnyWrap; 4793 } 4794 4795 bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) { 4796 // Here we check that I is in the header of the innermost loop containing I, 4797 // since we only deal with instructions in the loop header. The actual loop we 4798 // need to check later will come from an add recurrence, but getting that 4799 // requires computing the SCEV of the operands, which can be expensive. This 4800 // check we can do cheaply to rule out some cases early. 4801 Loop *InnermostContainingLoop = LI.getLoopFor(I->getParent()); 4802 if (InnermostContainingLoop == nullptr || 4803 InnermostContainingLoop->getHeader() != I->getParent()) 4804 return false; 4805 4806 // Only proceed if we can prove that I does not yield poison. 4807 if (!isKnownNotFullPoison(I)) return false; 4808 4809 // At this point we know that if I is executed, then it does not wrap 4810 // according to at least one of NSW or NUW. If I is not executed, then we do 4811 // not know if the calculation that I represents would wrap. Multiple 4812 // instructions can map to the same SCEV. If we apply NSW or NUW from I to 4813 // the SCEV, we must guarantee no wrapping for that SCEV also when it is 4814 // derived from other instructions that map to the same SCEV. We cannot make 4815 // that guarantee for cases where I is not executed. So we need to find the 4816 // loop that I is considered in relation to and prove that I is executed for 4817 // every iteration of that loop. That implies that the value that I 4818 // calculates does not wrap anywhere in the loop, so then we can apply the 4819 // flags to the SCEV. 4820 // 4821 // We check isLoopInvariant to disambiguate in case we are adding recurrences 4822 // from different loops, so that we know which loop to prove that I is 4823 // executed in. 4824 for (unsigned OpIndex = 0; OpIndex < I->getNumOperands(); ++OpIndex) { 4825 const SCEV *Op = getSCEV(I->getOperand(OpIndex)); 4826 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) { 4827 bool AllOtherOpsLoopInvariant = true; 4828 for (unsigned OtherOpIndex = 0; OtherOpIndex < I->getNumOperands(); 4829 ++OtherOpIndex) { 4830 if (OtherOpIndex != OpIndex) { 4831 const SCEV *OtherOp = getSCEV(I->getOperand(OtherOpIndex)); 4832 if (!isLoopInvariant(OtherOp, AddRec->getLoop())) { 4833 AllOtherOpsLoopInvariant = false; 4834 break; 4835 } 4836 } 4837 } 4838 if (AllOtherOpsLoopInvariant && 4839 isGuaranteedToExecuteForEveryIteration(I, AddRec->getLoop())) 4840 return true; 4841 } 4842 } 4843 return false; 4844 } 4845 4846 bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L) { 4847 // If we know that \c I can never be poison period, then that's enough. 4848 if (isSCEVExprNeverPoison(I)) 4849 return true; 4850 4851 // For an add recurrence specifically, we assume that infinite loops without 4852 // side effects are undefined behavior, and then reason as follows: 4853 // 4854 // If the add recurrence is poison in any iteration, it is poison on all 4855 // future iterations (since incrementing poison yields poison). If the result 4856 // of the add recurrence is fed into the loop latch condition and the loop 4857 // does not contain any throws or exiting blocks other than the latch, we now 4858 // have the ability to "choose" whether the backedge is taken or not (by 4859 // choosing a sufficiently evil value for the poison feeding into the branch) 4860 // for every iteration including and after the one in which \p I first became 4861 // poison. There are two possibilities (let's call the iteration in which \p 4862 // I first became poison as K): 4863 // 4864 // 1. In the set of iterations including and after K, the loop body executes 4865 // no side effects. In this case executing the backege an infinte number 4866 // of times will yield undefined behavior. 4867 // 4868 // 2. In the set of iterations including and after K, the loop body executes 4869 // at least one side effect. In this case, that specific instance of side 4870 // effect is control dependent on poison, which also yields undefined 4871 // behavior. 4872 4873 auto *ExitingBB = L->getExitingBlock(); 4874 auto *LatchBB = L->getLoopLatch(); 4875 if (!ExitingBB || !LatchBB || ExitingBB != LatchBB) 4876 return false; 4877 4878 SmallPtrSet<const Instruction *, 16> Pushed; 4879 SmallVector<const Instruction *, 8> PoisonStack; 4880 4881 // We start by assuming \c I, the post-inc add recurrence, is poison. Only 4882 // things that are known to be fully poison under that assumption go on the 4883 // PoisonStack. 4884 Pushed.insert(I); 4885 PoisonStack.push_back(I); 4886 4887 bool LatchControlDependentOnPoison = false; 4888 while (!PoisonStack.empty() && !LatchControlDependentOnPoison) { 4889 const Instruction *Poison = PoisonStack.pop_back_val(); 4890 4891 for (auto *PoisonUser : Poison->users()) { 4892 if (propagatesFullPoison(cast<Instruction>(PoisonUser))) { 4893 if (Pushed.insert(cast<Instruction>(PoisonUser)).second) 4894 PoisonStack.push_back(cast<Instruction>(PoisonUser)); 4895 } else if (auto *BI = dyn_cast<BranchInst>(PoisonUser)) { 4896 assert(BI->isConditional() && "Only possibility!"); 4897 if (BI->getParent() == LatchBB) { 4898 LatchControlDependentOnPoison = true; 4899 break; 4900 } 4901 } 4902 } 4903 } 4904 4905 return LatchControlDependentOnPoison && loopHasNoAbnormalExits(L); 4906 } 4907 4908 bool ScalarEvolution::loopHasNoAbnormalExits(const Loop *L) { 4909 auto Itr = LoopHasNoAbnormalExits.find(L); 4910 if (Itr == LoopHasNoAbnormalExits.end()) { 4911 auto NoAbnormalExitInBB = [&](BasicBlock *BB) { 4912 return all_of(*BB, [](Instruction &I) { 4913 return isGuaranteedToTransferExecutionToSuccessor(&I); 4914 }); 4915 }; 4916 4917 auto InsertPair = LoopHasNoAbnormalExits.insert( 4918 {L, all_of(L->getBlocks(), NoAbnormalExitInBB)}); 4919 assert(InsertPair.second && "We just checked!"); 4920 Itr = InsertPair.first; 4921 } 4922 4923 return Itr->second; 4924 } 4925 4926 const SCEV *ScalarEvolution::createSCEV(Value *V) { 4927 if (!isSCEVable(V->getType())) 4928 return getUnknown(V); 4929 4930 if (Instruction *I = dyn_cast<Instruction>(V)) { 4931 // Don't attempt to analyze instructions in blocks that aren't 4932 // reachable. Such instructions don't matter, and they aren't required 4933 // to obey basic rules for definitions dominating uses which this 4934 // analysis depends on. 4935 if (!DT.isReachableFromEntry(I->getParent())) 4936 return getUnknown(V); 4937 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) 4938 return getConstant(CI); 4939 else if (isa<ConstantPointerNull>(V)) 4940 return getZero(V->getType()); 4941 else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) 4942 return GA->isInterposable() ? getUnknown(V) : getSCEV(GA->getAliasee()); 4943 else if (!isa<ConstantExpr>(V)) 4944 return getUnknown(V); 4945 4946 Operator *U = cast<Operator>(V); 4947 if (auto BO = MatchBinaryOp(U, DT)) { 4948 switch (BO->Opcode) { 4949 case Instruction::Add: { 4950 // The simple thing to do would be to just call getSCEV on both operands 4951 // and call getAddExpr with the result. However if we're looking at a 4952 // bunch of things all added together, this can be quite inefficient, 4953 // because it leads to N-1 getAddExpr calls for N ultimate operands. 4954 // Instead, gather up all the operands and make a single getAddExpr call. 4955 // LLVM IR canonical form means we need only traverse the left operands. 4956 SmallVector<const SCEV *, 4> AddOps; 4957 do { 4958 if (BO->Op) { 4959 if (auto *OpSCEV = getExistingSCEV(BO->Op)) { 4960 AddOps.push_back(OpSCEV); 4961 break; 4962 } 4963 4964 // If a NUW or NSW flag can be applied to the SCEV for this 4965 // addition, then compute the SCEV for this addition by itself 4966 // with a separate call to getAddExpr. We need to do that 4967 // instead of pushing the operands of the addition onto AddOps, 4968 // since the flags are only known to apply to this particular 4969 // addition - they may not apply to other additions that can be 4970 // formed with operands from AddOps. 4971 const SCEV *RHS = getSCEV(BO->RHS); 4972 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(BO->Op); 4973 if (Flags != SCEV::FlagAnyWrap) { 4974 const SCEV *LHS = getSCEV(BO->LHS); 4975 if (BO->Opcode == Instruction::Sub) 4976 AddOps.push_back(getMinusSCEV(LHS, RHS, Flags)); 4977 else 4978 AddOps.push_back(getAddExpr(LHS, RHS, Flags)); 4979 break; 4980 } 4981 } 4982 4983 if (BO->Opcode == Instruction::Sub) 4984 AddOps.push_back(getNegativeSCEV(getSCEV(BO->RHS))); 4985 else 4986 AddOps.push_back(getSCEV(BO->RHS)); 4987 4988 auto NewBO = MatchBinaryOp(BO->LHS, DT); 4989 if (!NewBO || (NewBO->Opcode != Instruction::Add && 4990 NewBO->Opcode != Instruction::Sub)) { 4991 AddOps.push_back(getSCEV(BO->LHS)); 4992 break; 4993 } 4994 BO = NewBO; 4995 } while (true); 4996 4997 return getAddExpr(AddOps); 4998 } 4999 5000 case Instruction::Mul: { 5001 SmallVector<const SCEV *, 4> MulOps; 5002 do { 5003 if (BO->Op) { 5004 if (auto *OpSCEV = getExistingSCEV(BO->Op)) { 5005 MulOps.push_back(OpSCEV); 5006 break; 5007 } 5008 5009 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(BO->Op); 5010 if (Flags != SCEV::FlagAnyWrap) { 5011 MulOps.push_back( 5012 getMulExpr(getSCEV(BO->LHS), getSCEV(BO->RHS), Flags)); 5013 break; 5014 } 5015 } 5016 5017 MulOps.push_back(getSCEV(BO->RHS)); 5018 auto NewBO = MatchBinaryOp(BO->LHS, DT); 5019 if (!NewBO || NewBO->Opcode != Instruction::Mul) { 5020 MulOps.push_back(getSCEV(BO->LHS)); 5021 break; 5022 } 5023 BO = NewBO; 5024 } while (true); 5025 5026 return getMulExpr(MulOps); 5027 } 5028 case Instruction::UDiv: 5029 return getUDivExpr(getSCEV(BO->LHS), getSCEV(BO->RHS)); 5030 case Instruction::Sub: { 5031 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap; 5032 if (BO->Op) 5033 Flags = getNoWrapFlagsFromUB(BO->Op); 5034 return getMinusSCEV(getSCEV(BO->LHS), getSCEV(BO->RHS), Flags); 5035 } 5036 case Instruction::And: 5037 // For an expression like x&255 that merely masks off the high bits, 5038 // use zext(trunc(x)) as the SCEV expression. 5039 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) { 5040 if (CI->isNullValue()) 5041 return getSCEV(BO->RHS); 5042 if (CI->isAllOnesValue()) 5043 return getSCEV(BO->LHS); 5044 const APInt &A = CI->getValue(); 5045 5046 // Instcombine's ShrinkDemandedConstant may strip bits out of 5047 // constants, obscuring what would otherwise be a low-bits mask. 5048 // Use computeKnownBits to compute what ShrinkDemandedConstant 5049 // knew about to reconstruct a low-bits mask value. 5050 unsigned LZ = A.countLeadingZeros(); 5051 unsigned TZ = A.countTrailingZeros(); 5052 unsigned BitWidth = A.getBitWidth(); 5053 APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0); 5054 computeKnownBits(BO->LHS, KnownZero, KnownOne, getDataLayout(), 5055 0, &AC, nullptr, &DT); 5056 5057 APInt EffectiveMask = 5058 APInt::getLowBitsSet(BitWidth, BitWidth - LZ - TZ).shl(TZ); 5059 if ((LZ != 0 || TZ != 0) && !((~A & ~KnownZero) & EffectiveMask)) { 5060 const SCEV *MulCount = getConstant(ConstantInt::get( 5061 getContext(), APInt::getOneBitSet(BitWidth, TZ))); 5062 return getMulExpr( 5063 getZeroExtendExpr( 5064 getTruncateExpr( 5065 getUDivExactExpr(getSCEV(BO->LHS), MulCount), 5066 IntegerType::get(getContext(), BitWidth - LZ - TZ)), 5067 BO->LHS->getType()), 5068 MulCount); 5069 } 5070 } 5071 break; 5072 5073 case Instruction::Or: 5074 // If the RHS of the Or is a constant, we may have something like: 5075 // X*4+1 which got turned into X*4|1. Handle this as an Add so loop 5076 // optimizations will transparently handle this case. 5077 // 5078 // In order for this transformation to be safe, the LHS must be of the 5079 // form X*(2^n) and the Or constant must be less than 2^n. 5080 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) { 5081 const SCEV *LHS = getSCEV(BO->LHS); 5082 const APInt &CIVal = CI->getValue(); 5083 if (GetMinTrailingZeros(LHS) >= 5084 (CIVal.getBitWidth() - CIVal.countLeadingZeros())) { 5085 // Build a plain add SCEV. 5086 const SCEV *S = getAddExpr(LHS, getSCEV(CI)); 5087 // If the LHS of the add was an addrec and it has no-wrap flags, 5088 // transfer the no-wrap flags, since an or won't introduce a wrap. 5089 if (const SCEVAddRecExpr *NewAR = dyn_cast<SCEVAddRecExpr>(S)) { 5090 const SCEVAddRecExpr *OldAR = cast<SCEVAddRecExpr>(LHS); 5091 const_cast<SCEVAddRecExpr *>(NewAR)->setNoWrapFlags( 5092 OldAR->getNoWrapFlags()); 5093 } 5094 return S; 5095 } 5096 } 5097 break; 5098 5099 case Instruction::Xor: 5100 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) { 5101 // If the RHS of xor is -1, then this is a not operation. 5102 if (CI->isAllOnesValue()) 5103 return getNotSCEV(getSCEV(BO->LHS)); 5104 5105 // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask. 5106 // This is a variant of the check for xor with -1, and it handles 5107 // the case where instcombine has trimmed non-demanded bits out 5108 // of an xor with -1. 5109 if (auto *LBO = dyn_cast<BinaryOperator>(BO->LHS)) 5110 if (ConstantInt *LCI = dyn_cast<ConstantInt>(LBO->getOperand(1))) 5111 if (LBO->getOpcode() == Instruction::And && 5112 LCI->getValue() == CI->getValue()) 5113 if (const SCEVZeroExtendExpr *Z = 5114 dyn_cast<SCEVZeroExtendExpr>(getSCEV(BO->LHS))) { 5115 Type *UTy = BO->LHS->getType(); 5116 const SCEV *Z0 = Z->getOperand(); 5117 Type *Z0Ty = Z0->getType(); 5118 unsigned Z0TySize = getTypeSizeInBits(Z0Ty); 5119 5120 // If C is a low-bits mask, the zero extend is serving to 5121 // mask off the high bits. Complement the operand and 5122 // re-apply the zext. 5123 if (APIntOps::isMask(Z0TySize, CI->getValue())) 5124 return getZeroExtendExpr(getNotSCEV(Z0), UTy); 5125 5126 // If C is a single bit, it may be in the sign-bit position 5127 // before the zero-extend. In this case, represent the xor 5128 // using an add, which is equivalent, and re-apply the zext. 5129 APInt Trunc = CI->getValue().trunc(Z0TySize); 5130 if (Trunc.zext(getTypeSizeInBits(UTy)) == CI->getValue() && 5131 Trunc.isSignBit()) 5132 return getZeroExtendExpr(getAddExpr(Z0, getConstant(Trunc)), 5133 UTy); 5134 } 5135 } 5136 break; 5137 5138 case Instruction::Shl: 5139 // Turn shift left of a constant amount into a multiply. 5140 if (ConstantInt *SA = dyn_cast<ConstantInt>(BO->RHS)) { 5141 uint32_t BitWidth = cast<IntegerType>(SA->getType())->getBitWidth(); 5142 5143 // If the shift count is not less than the bitwidth, the result of 5144 // the shift is undefined. Don't try to analyze it, because the 5145 // resolution chosen here may differ from the resolution chosen in 5146 // other parts of the compiler. 5147 if (SA->getValue().uge(BitWidth)) 5148 break; 5149 5150 // It is currently not resolved how to interpret NSW for left 5151 // shift by BitWidth - 1, so we avoid applying flags in that 5152 // case. Remove this check (or this comment) once the situation 5153 // is resolved. See 5154 // http://lists.llvm.org/pipermail/llvm-dev/2015-April/084195.html 5155 // and http://reviews.llvm.org/D8890 . 5156 auto Flags = SCEV::FlagAnyWrap; 5157 if (BO->Op && SA->getValue().ult(BitWidth - 1)) 5158 Flags = getNoWrapFlagsFromUB(BO->Op); 5159 5160 Constant *X = ConstantInt::get(getContext(), 5161 APInt::getOneBitSet(BitWidth, SA->getZExtValue())); 5162 return getMulExpr(getSCEV(BO->LHS), getSCEV(X), Flags); 5163 } 5164 break; 5165 5166 case Instruction::AShr: 5167 // For a two-shift sext-inreg, use sext(trunc(x)) as the SCEV expression. 5168 if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) 5169 if (Operator *L = dyn_cast<Operator>(BO->LHS)) 5170 if (L->getOpcode() == Instruction::Shl && 5171 L->getOperand(1) == BO->RHS) { 5172 uint64_t BitWidth = getTypeSizeInBits(BO->LHS->getType()); 5173 5174 // If the shift count is not less than the bitwidth, the result of 5175 // the shift is undefined. Don't try to analyze it, because the 5176 // resolution chosen here may differ from the resolution chosen in 5177 // other parts of the compiler. 5178 if (CI->getValue().uge(BitWidth)) 5179 break; 5180 5181 uint64_t Amt = BitWidth - CI->getZExtValue(); 5182 if (Amt == BitWidth) 5183 return getSCEV(L->getOperand(0)); // shift by zero --> noop 5184 return getSignExtendExpr( 5185 getTruncateExpr(getSCEV(L->getOperand(0)), 5186 IntegerType::get(getContext(), Amt)), 5187 BO->LHS->getType()); 5188 } 5189 break; 5190 } 5191 } 5192 5193 switch (U->getOpcode()) { 5194 case Instruction::Trunc: 5195 return getTruncateExpr(getSCEV(U->getOperand(0)), U->getType()); 5196 5197 case Instruction::ZExt: 5198 return getZeroExtendExpr(getSCEV(U->getOperand(0)), U->getType()); 5199 5200 case Instruction::SExt: 5201 return getSignExtendExpr(getSCEV(U->getOperand(0)), U->getType()); 5202 5203 case Instruction::BitCast: 5204 // BitCasts are no-op casts so we just eliminate the cast. 5205 if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType())) 5206 return getSCEV(U->getOperand(0)); 5207 break; 5208 5209 // It's tempting to handle inttoptr and ptrtoint as no-ops, however this can 5210 // lead to pointer expressions which cannot safely be expanded to GEPs, 5211 // because ScalarEvolution doesn't respect the GEP aliasing rules when 5212 // simplifying integer expressions. 5213 5214 case Instruction::GetElementPtr: 5215 return createNodeForGEP(cast<GEPOperator>(U)); 5216 5217 case Instruction::PHI: 5218 return createNodeForPHI(cast<PHINode>(U)); 5219 5220 case Instruction::Select: 5221 // U can also be a select constant expr, which let fall through. Since 5222 // createNodeForSelect only works for a condition that is an `ICmpInst`, and 5223 // constant expressions cannot have instructions as operands, we'd have 5224 // returned getUnknown for a select constant expressions anyway. 5225 if (isa<Instruction>(U)) 5226 return createNodeForSelectOrPHI(cast<Instruction>(U), U->getOperand(0), 5227 U->getOperand(1), U->getOperand(2)); 5228 break; 5229 5230 case Instruction::Call: 5231 case Instruction::Invoke: 5232 if (Value *RV = CallSite(U).getReturnedArgOperand()) 5233 return getSCEV(RV); 5234 break; 5235 } 5236 5237 return getUnknown(V); 5238 } 5239 5240 5241 5242 //===----------------------------------------------------------------------===// 5243 // Iteration Count Computation Code 5244 // 5245 5246 unsigned ScalarEvolution::getSmallConstantTripCount(Loop *L) { 5247 if (BasicBlock *ExitingBB = L->getExitingBlock()) 5248 return getSmallConstantTripCount(L, ExitingBB); 5249 5250 // No trip count information for multiple exits. 5251 return 0; 5252 } 5253 5254 unsigned ScalarEvolution::getSmallConstantTripCount(Loop *L, 5255 BasicBlock *ExitingBlock) { 5256 assert(ExitingBlock && "Must pass a non-null exiting block!"); 5257 assert(L->isLoopExiting(ExitingBlock) && 5258 "Exiting block must actually branch out of the loop!"); 5259 const SCEVConstant *ExitCount = 5260 dyn_cast<SCEVConstant>(getExitCount(L, ExitingBlock)); 5261 if (!ExitCount) 5262 return 0; 5263 5264 ConstantInt *ExitConst = ExitCount->getValue(); 5265 5266 // Guard against huge trip counts. 5267 if (ExitConst->getValue().getActiveBits() > 32) 5268 return 0; 5269 5270 // In case of integer overflow, this returns 0, which is correct. 5271 return ((unsigned)ExitConst->getZExtValue()) + 1; 5272 } 5273 5274 unsigned ScalarEvolution::getSmallConstantTripMultiple(Loop *L) { 5275 if (BasicBlock *ExitingBB = L->getExitingBlock()) 5276 return getSmallConstantTripMultiple(L, ExitingBB); 5277 5278 // No trip multiple information for multiple exits. 5279 return 0; 5280 } 5281 5282 /// Returns the largest constant divisor of the trip count of this loop as a 5283 /// normal unsigned value, if possible. This means that the actual trip count is 5284 /// always a multiple of the returned value (don't forget the trip count could 5285 /// very well be zero as well!). 5286 /// 5287 /// Returns 1 if the trip count is unknown or not guaranteed to be the 5288 /// multiple of a constant (which is also the case if the trip count is simply 5289 /// constant, use getSmallConstantTripCount for that case), Will also return 1 5290 /// if the trip count is very large (>= 2^32). 5291 /// 5292 /// As explained in the comments for getSmallConstantTripCount, this assumes 5293 /// that control exits the loop via ExitingBlock. 5294 unsigned 5295 ScalarEvolution::getSmallConstantTripMultiple(Loop *L, 5296 BasicBlock *ExitingBlock) { 5297 assert(ExitingBlock && "Must pass a non-null exiting block!"); 5298 assert(L->isLoopExiting(ExitingBlock) && 5299 "Exiting block must actually branch out of the loop!"); 5300 const SCEV *ExitCount = getExitCount(L, ExitingBlock); 5301 if (ExitCount == getCouldNotCompute()) 5302 return 1; 5303 5304 // Get the trip count from the BE count by adding 1. 5305 const SCEV *TCMul = getAddExpr(ExitCount, getOne(ExitCount->getType())); 5306 // FIXME: SCEV distributes multiplication as V1*C1 + V2*C1. We could attempt 5307 // to factor simple cases. 5308 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(TCMul)) 5309 TCMul = Mul->getOperand(0); 5310 5311 const SCEVConstant *MulC = dyn_cast<SCEVConstant>(TCMul); 5312 if (!MulC) 5313 return 1; 5314 5315 ConstantInt *Result = MulC->getValue(); 5316 5317 // Guard against huge trip counts (this requires checking 5318 // for zero to handle the case where the trip count == -1 and the 5319 // addition wraps). 5320 if (!Result || Result->getValue().getActiveBits() > 32 || 5321 Result->getValue().getActiveBits() == 0) 5322 return 1; 5323 5324 return (unsigned)Result->getZExtValue(); 5325 } 5326 5327 /// Get the expression for the number of loop iterations for which this loop is 5328 /// guaranteed not to exit via ExitingBlock. Otherwise return 5329 /// SCEVCouldNotCompute. 5330 const SCEV *ScalarEvolution::getExitCount(Loop *L, BasicBlock *ExitingBlock) { 5331 return getBackedgeTakenInfo(L).getExact(ExitingBlock, this); 5332 } 5333 5334 const SCEV * 5335 ScalarEvolution::getPredicatedBackedgeTakenCount(const Loop *L, 5336 SCEVUnionPredicate &Preds) { 5337 return getPredicatedBackedgeTakenInfo(L).getExact(this, &Preds); 5338 } 5339 5340 const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L) { 5341 return getBackedgeTakenInfo(L).getExact(this); 5342 } 5343 5344 /// Similar to getBackedgeTakenCount, except return the least SCEV value that is 5345 /// known never to be less than the actual backedge taken count. 5346 const SCEV *ScalarEvolution::getMaxBackedgeTakenCount(const Loop *L) { 5347 return getBackedgeTakenInfo(L).getMax(this); 5348 } 5349 5350 /// Push PHI nodes in the header of the given loop onto the given Worklist. 5351 static void 5352 PushLoopPHIs(const Loop *L, SmallVectorImpl<Instruction *> &Worklist) { 5353 BasicBlock *Header = L->getHeader(); 5354 5355 // Push all Loop-header PHIs onto the Worklist stack. 5356 for (BasicBlock::iterator I = Header->begin(); 5357 PHINode *PN = dyn_cast<PHINode>(I); ++I) 5358 Worklist.push_back(PN); 5359 } 5360 5361 const ScalarEvolution::BackedgeTakenInfo & 5362 ScalarEvolution::getPredicatedBackedgeTakenInfo(const Loop *L) { 5363 auto &BTI = getBackedgeTakenInfo(L); 5364 if (BTI.hasFullInfo()) 5365 return BTI; 5366 5367 auto Pair = PredicatedBackedgeTakenCounts.insert({L, BackedgeTakenInfo()}); 5368 5369 if (!Pair.second) 5370 return Pair.first->second; 5371 5372 BackedgeTakenInfo Result = 5373 computeBackedgeTakenCount(L, /*AllowPredicates=*/true); 5374 5375 return PredicatedBackedgeTakenCounts.find(L)->second = Result; 5376 } 5377 5378 const ScalarEvolution::BackedgeTakenInfo & 5379 ScalarEvolution::getBackedgeTakenInfo(const Loop *L) { 5380 // Initially insert an invalid entry for this loop. If the insertion 5381 // succeeds, proceed to actually compute a backedge-taken count and 5382 // update the value. The temporary CouldNotCompute value tells SCEV 5383 // code elsewhere that it shouldn't attempt to request a new 5384 // backedge-taken count, which could result in infinite recursion. 5385 std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair = 5386 BackedgeTakenCounts.insert({L, BackedgeTakenInfo()}); 5387 if (!Pair.second) 5388 return Pair.first->second; 5389 5390 // computeBackedgeTakenCount may allocate memory for its result. Inserting it 5391 // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result 5392 // must be cleared in this scope. 5393 BackedgeTakenInfo Result = computeBackedgeTakenCount(L); 5394 5395 if (Result.getExact(this) != getCouldNotCompute()) { 5396 assert(isLoopInvariant(Result.getExact(this), L) && 5397 isLoopInvariant(Result.getMax(this), L) && 5398 "Computed backedge-taken count isn't loop invariant for loop!"); 5399 ++NumTripCountsComputed; 5400 } 5401 else if (Result.getMax(this) == getCouldNotCompute() && 5402 isa<PHINode>(L->getHeader()->begin())) { 5403 // Only count loops that have phi nodes as not being computable. 5404 ++NumTripCountsNotComputed; 5405 } 5406 5407 // Now that we know more about the trip count for this loop, forget any 5408 // existing SCEV values for PHI nodes in this loop since they are only 5409 // conservative estimates made without the benefit of trip count 5410 // information. This is similar to the code in forgetLoop, except that 5411 // it handles SCEVUnknown PHI nodes specially. 5412 if (Result.hasAnyInfo()) { 5413 SmallVector<Instruction *, 16> Worklist; 5414 PushLoopPHIs(L, Worklist); 5415 5416 SmallPtrSet<Instruction *, 8> Visited; 5417 while (!Worklist.empty()) { 5418 Instruction *I = Worklist.pop_back_val(); 5419 if (!Visited.insert(I).second) 5420 continue; 5421 5422 ValueExprMapType::iterator It = 5423 ValueExprMap.find_as(static_cast<Value *>(I)); 5424 if (It != ValueExprMap.end()) { 5425 const SCEV *Old = It->second; 5426 5427 // SCEVUnknown for a PHI either means that it has an unrecognized 5428 // structure, or it's a PHI that's in the progress of being computed 5429 // by createNodeForPHI. In the former case, additional loop trip 5430 // count information isn't going to change anything. In the later 5431 // case, createNodeForPHI will perform the necessary updates on its 5432 // own when it gets to that point. 5433 if (!isa<PHINode>(I) || !isa<SCEVUnknown>(Old)) { 5434 forgetMemoizedResults(Old); 5435 ValueExprMap.erase(It); 5436 } 5437 if (PHINode *PN = dyn_cast<PHINode>(I)) 5438 ConstantEvolutionLoopExitValue.erase(PN); 5439 } 5440 5441 PushDefUseChildren(I, Worklist); 5442 } 5443 } 5444 5445 // Re-lookup the insert position, since the call to 5446 // computeBackedgeTakenCount above could result in a 5447 // recusive call to getBackedgeTakenInfo (on a different 5448 // loop), which would invalidate the iterator computed 5449 // earlier. 5450 return BackedgeTakenCounts.find(L)->second = Result; 5451 } 5452 5453 void ScalarEvolution::forgetLoop(const Loop *L) { 5454 // Drop any stored trip count value. 5455 auto RemoveLoopFromBackedgeMap = 5456 [L](DenseMap<const Loop *, BackedgeTakenInfo> &Map) { 5457 auto BTCPos = Map.find(L); 5458 if (BTCPos != Map.end()) { 5459 BTCPos->second.clear(); 5460 Map.erase(BTCPos); 5461 } 5462 }; 5463 5464 RemoveLoopFromBackedgeMap(BackedgeTakenCounts); 5465 RemoveLoopFromBackedgeMap(PredicatedBackedgeTakenCounts); 5466 5467 // Drop information about expressions based on loop-header PHIs. 5468 SmallVector<Instruction *, 16> Worklist; 5469 PushLoopPHIs(L, Worklist); 5470 5471 SmallPtrSet<Instruction *, 8> Visited; 5472 while (!Worklist.empty()) { 5473 Instruction *I = Worklist.pop_back_val(); 5474 if (!Visited.insert(I).second) 5475 continue; 5476 5477 ValueExprMapType::iterator It = 5478 ValueExprMap.find_as(static_cast<Value *>(I)); 5479 if (It != ValueExprMap.end()) { 5480 forgetMemoizedResults(It->second); 5481 ValueExprMap.erase(It); 5482 if (PHINode *PN = dyn_cast<PHINode>(I)) 5483 ConstantEvolutionLoopExitValue.erase(PN); 5484 } 5485 5486 PushDefUseChildren(I, Worklist); 5487 } 5488 5489 // Forget all contained loops too, to avoid dangling entries in the 5490 // ValuesAtScopes map. 5491 for (Loop *I : *L) 5492 forgetLoop(I); 5493 5494 LoopHasNoAbnormalExits.erase(L); 5495 } 5496 5497 void ScalarEvolution::forgetValue(Value *V) { 5498 Instruction *I = dyn_cast<Instruction>(V); 5499 if (!I) return; 5500 5501 // Drop information about expressions based on loop-header PHIs. 5502 SmallVector<Instruction *, 16> Worklist; 5503 Worklist.push_back(I); 5504 5505 SmallPtrSet<Instruction *, 8> Visited; 5506 while (!Worklist.empty()) { 5507 I = Worklist.pop_back_val(); 5508 if (!Visited.insert(I).second) 5509 continue; 5510 5511 ValueExprMapType::iterator It = 5512 ValueExprMap.find_as(static_cast<Value *>(I)); 5513 if (It != ValueExprMap.end()) { 5514 forgetMemoizedResults(It->second); 5515 ValueExprMap.erase(It); 5516 if (PHINode *PN = dyn_cast<PHINode>(I)) 5517 ConstantEvolutionLoopExitValue.erase(PN); 5518 } 5519 5520 PushDefUseChildren(I, Worklist); 5521 } 5522 } 5523 5524 /// Get the exact loop backedge taken count considering all loop exits. A 5525 /// computable result can only be returned for loops with a single exit. 5526 /// Returning the minimum taken count among all exits is incorrect because one 5527 /// of the loop's exit limit's may have been skipped. howFarToZero assumes that 5528 /// the limit of each loop test is never skipped. This is a valid assumption as 5529 /// long as the loop exits via that test. For precise results, it is the 5530 /// caller's responsibility to specify the relevant loop exit using 5531 /// getExact(ExitingBlock, SE). 5532 const SCEV * 5533 ScalarEvolution::BackedgeTakenInfo::getExact( 5534 ScalarEvolution *SE, SCEVUnionPredicate *Preds) const { 5535 // If any exits were not computable, the loop is not computable. 5536 if (!ExitNotTaken.isCompleteList()) return SE->getCouldNotCompute(); 5537 5538 // We need exactly one computable exit. 5539 if (!ExitNotTaken.ExitingBlock) return SE->getCouldNotCompute(); 5540 assert(ExitNotTaken.ExactNotTaken && "uninitialized not-taken info"); 5541 5542 const SCEV *BECount = nullptr; 5543 for (auto &ENT : ExitNotTaken) { 5544 assert(ENT.ExactNotTaken != SE->getCouldNotCompute() && "bad exit SCEV"); 5545 5546 if (!BECount) 5547 BECount = ENT.ExactNotTaken; 5548 else if (BECount != ENT.ExactNotTaken) 5549 return SE->getCouldNotCompute(); 5550 if (Preds && ENT.getPred()) 5551 Preds->add(ENT.getPred()); 5552 5553 assert((Preds || ENT.hasAlwaysTruePred()) && 5554 "Predicate should be always true!"); 5555 } 5556 5557 assert(BECount && "Invalid not taken count for loop exit"); 5558 return BECount; 5559 } 5560 5561 /// Get the exact not taken count for this loop exit. 5562 const SCEV * 5563 ScalarEvolution::BackedgeTakenInfo::getExact(BasicBlock *ExitingBlock, 5564 ScalarEvolution *SE) const { 5565 for (auto &ENT : ExitNotTaken) 5566 if (ENT.ExitingBlock == ExitingBlock && ENT.hasAlwaysTruePred()) 5567 return ENT.ExactNotTaken; 5568 5569 return SE->getCouldNotCompute(); 5570 } 5571 5572 /// getMax - Get the max backedge taken count for the loop. 5573 const SCEV * 5574 ScalarEvolution::BackedgeTakenInfo::getMax(ScalarEvolution *SE) const { 5575 for (auto &ENT : ExitNotTaken) 5576 if (!ENT.hasAlwaysTruePred()) 5577 return SE->getCouldNotCompute(); 5578 5579 return Max ? Max : SE->getCouldNotCompute(); 5580 } 5581 5582 bool ScalarEvolution::BackedgeTakenInfo::hasOperand(const SCEV *S, 5583 ScalarEvolution *SE) const { 5584 if (Max && Max != SE->getCouldNotCompute() && SE->hasOperand(Max, S)) 5585 return true; 5586 5587 if (!ExitNotTaken.ExitingBlock) 5588 return false; 5589 5590 for (auto &ENT : ExitNotTaken) 5591 if (ENT.ExactNotTaken != SE->getCouldNotCompute() && 5592 SE->hasOperand(ENT.ExactNotTaken, S)) 5593 return true; 5594 5595 return false; 5596 } 5597 5598 /// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each 5599 /// computable exit into a persistent ExitNotTakenInfo array. 5600 ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo( 5601 SmallVectorImpl<EdgeInfo> &ExitCounts, bool Complete, const SCEV *MaxCount) 5602 : Max(MaxCount) { 5603 5604 if (!Complete) 5605 ExitNotTaken.setIncomplete(); 5606 5607 unsigned NumExits = ExitCounts.size(); 5608 if (NumExits == 0) return; 5609 5610 ExitNotTaken.ExitingBlock = ExitCounts[0].ExitBlock; 5611 ExitNotTaken.ExactNotTaken = ExitCounts[0].Taken; 5612 5613 // Determine the number of ExitNotTakenExtras structures that we need. 5614 unsigned ExtraInfoSize = 0; 5615 if (NumExits > 1) 5616 ExtraInfoSize = 1 + std::count_if(std::next(ExitCounts.begin()), 5617 ExitCounts.end(), [](EdgeInfo &Entry) { 5618 return !Entry.Pred.isAlwaysTrue(); 5619 }); 5620 else if (!ExitCounts[0].Pred.isAlwaysTrue()) 5621 ExtraInfoSize = 1; 5622 5623 ExitNotTakenExtras *ENT = nullptr; 5624 5625 // Allocate the ExitNotTakenExtras structures and initialize the first 5626 // element (ExitNotTaken). 5627 if (ExtraInfoSize > 0) { 5628 ENT = new ExitNotTakenExtras[ExtraInfoSize]; 5629 ExitNotTaken.ExtraInfo = &ENT[0]; 5630 *ExitNotTaken.getPred() = std::move(ExitCounts[0].Pred); 5631 } 5632 5633 if (NumExits == 1) 5634 return; 5635 5636 assert(ENT && "ExitNotTakenExtras is NULL while having more than one exit"); 5637 5638 auto &Exits = ExitNotTaken.ExtraInfo->Exits; 5639 5640 // Handle the rare case of multiple computable exits. 5641 for (unsigned i = 1, PredPos = 1; i < NumExits; ++i) { 5642 ExitNotTakenExtras *Ptr = nullptr; 5643 if (!ExitCounts[i].Pred.isAlwaysTrue()) { 5644 Ptr = &ENT[PredPos++]; 5645 Ptr->Pred = std::move(ExitCounts[i].Pred); 5646 } 5647 5648 Exits.emplace_back(ExitCounts[i].ExitBlock, ExitCounts[i].Taken, Ptr); 5649 } 5650 } 5651 5652 /// Invalidate this result and free the ExitNotTakenInfo array. 5653 void ScalarEvolution::BackedgeTakenInfo::clear() { 5654 ExitNotTaken.ExitingBlock = nullptr; 5655 ExitNotTaken.ExactNotTaken = nullptr; 5656 delete[] ExitNotTaken.ExtraInfo; 5657 } 5658 5659 /// Compute the number of times the backedge of the specified loop will execute. 5660 ScalarEvolution::BackedgeTakenInfo 5661 ScalarEvolution::computeBackedgeTakenCount(const Loop *L, 5662 bool AllowPredicates) { 5663 SmallVector<BasicBlock *, 8> ExitingBlocks; 5664 L->getExitingBlocks(ExitingBlocks); 5665 5666 SmallVector<EdgeInfo, 4> ExitCounts; 5667 bool CouldComputeBECount = true; 5668 BasicBlock *Latch = L->getLoopLatch(); // may be NULL. 5669 const SCEV *MustExitMaxBECount = nullptr; 5670 const SCEV *MayExitMaxBECount = nullptr; 5671 5672 // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts 5673 // and compute maxBECount. 5674 // Do a union of all the predicates here. 5675 for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) { 5676 BasicBlock *ExitBB = ExitingBlocks[i]; 5677 ExitLimit EL = computeExitLimit(L, ExitBB, AllowPredicates); 5678 5679 assert((AllowPredicates || EL.Pred.isAlwaysTrue()) && 5680 "Predicated exit limit when predicates are not allowed!"); 5681 5682 // 1. For each exit that can be computed, add an entry to ExitCounts. 5683 // CouldComputeBECount is true only if all exits can be computed. 5684 if (EL.Exact == getCouldNotCompute()) 5685 // We couldn't compute an exact value for this exit, so 5686 // we won't be able to compute an exact value for the loop. 5687 CouldComputeBECount = false; 5688 else 5689 ExitCounts.emplace_back(EdgeInfo(ExitBB, EL.Exact, EL.Pred)); 5690 5691 // 2. Derive the loop's MaxBECount from each exit's max number of 5692 // non-exiting iterations. Partition the loop exits into two kinds: 5693 // LoopMustExits and LoopMayExits. 5694 // 5695 // If the exit dominates the loop latch, it is a LoopMustExit otherwise it 5696 // is a LoopMayExit. If any computable LoopMustExit is found, then 5697 // MaxBECount is the minimum EL.Max of computable LoopMustExits. Otherwise, 5698 // MaxBECount is conservatively the maximum EL.Max, where CouldNotCompute is 5699 // considered greater than any computable EL.Max. 5700 if (EL.Max != getCouldNotCompute() && Latch && 5701 DT.dominates(ExitBB, Latch)) { 5702 if (!MustExitMaxBECount) 5703 MustExitMaxBECount = EL.Max; 5704 else { 5705 MustExitMaxBECount = 5706 getUMinFromMismatchedTypes(MustExitMaxBECount, EL.Max); 5707 } 5708 } else if (MayExitMaxBECount != getCouldNotCompute()) { 5709 if (!MayExitMaxBECount || EL.Max == getCouldNotCompute()) 5710 MayExitMaxBECount = EL.Max; 5711 else { 5712 MayExitMaxBECount = 5713 getUMaxFromMismatchedTypes(MayExitMaxBECount, EL.Max); 5714 } 5715 } 5716 } 5717 const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount : 5718 (MayExitMaxBECount ? MayExitMaxBECount : getCouldNotCompute()); 5719 return BackedgeTakenInfo(ExitCounts, CouldComputeBECount, MaxBECount); 5720 } 5721 5722 ScalarEvolution::ExitLimit 5723 ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock, 5724 bool AllowPredicates) { 5725 5726 // Okay, we've chosen an exiting block. See what condition causes us to exit 5727 // at this block and remember the exit block and whether all other targets 5728 // lead to the loop header. 5729 bool MustExecuteLoopHeader = true; 5730 BasicBlock *Exit = nullptr; 5731 for (auto *SBB : successors(ExitingBlock)) 5732 if (!L->contains(SBB)) { 5733 if (Exit) // Multiple exit successors. 5734 return getCouldNotCompute(); 5735 Exit = SBB; 5736 } else if (SBB != L->getHeader()) { 5737 MustExecuteLoopHeader = false; 5738 } 5739 5740 // At this point, we know we have a conditional branch that determines whether 5741 // the loop is exited. However, we don't know if the branch is executed each 5742 // time through the loop. If not, then the execution count of the branch will 5743 // not be equal to the trip count of the loop. 5744 // 5745 // Currently we check for this by checking to see if the Exit branch goes to 5746 // the loop header. If so, we know it will always execute the same number of 5747 // times as the loop. We also handle the case where the exit block *is* the 5748 // loop header. This is common for un-rotated loops. 5749 // 5750 // If both of those tests fail, walk up the unique predecessor chain to the 5751 // header, stopping if there is an edge that doesn't exit the loop. If the 5752 // header is reached, the execution count of the branch will be equal to the 5753 // trip count of the loop. 5754 // 5755 // More extensive analysis could be done to handle more cases here. 5756 // 5757 if (!MustExecuteLoopHeader && ExitingBlock != L->getHeader()) { 5758 // The simple checks failed, try climbing the unique predecessor chain 5759 // up to the header. 5760 bool Ok = false; 5761 for (BasicBlock *BB = ExitingBlock; BB; ) { 5762 BasicBlock *Pred = BB->getUniquePredecessor(); 5763 if (!Pred) 5764 return getCouldNotCompute(); 5765 TerminatorInst *PredTerm = Pred->getTerminator(); 5766 for (const BasicBlock *PredSucc : PredTerm->successors()) { 5767 if (PredSucc == BB) 5768 continue; 5769 // If the predecessor has a successor that isn't BB and isn't 5770 // outside the loop, assume the worst. 5771 if (L->contains(PredSucc)) 5772 return getCouldNotCompute(); 5773 } 5774 if (Pred == L->getHeader()) { 5775 Ok = true; 5776 break; 5777 } 5778 BB = Pred; 5779 } 5780 if (!Ok) 5781 return getCouldNotCompute(); 5782 } 5783 5784 bool IsOnlyExit = (L->getExitingBlock() != nullptr); 5785 TerminatorInst *Term = ExitingBlock->getTerminator(); 5786 if (BranchInst *BI = dyn_cast<BranchInst>(Term)) { 5787 assert(BI->isConditional() && "If unconditional, it can't be in loop!"); 5788 // Proceed to the next level to examine the exit condition expression. 5789 return computeExitLimitFromCond( 5790 L, BI->getCondition(), BI->getSuccessor(0), BI->getSuccessor(1), 5791 /*ControlsExit=*/IsOnlyExit, AllowPredicates); 5792 } 5793 5794 if (SwitchInst *SI = dyn_cast<SwitchInst>(Term)) 5795 return computeExitLimitFromSingleExitSwitch(L, SI, Exit, 5796 /*ControlsExit=*/IsOnlyExit); 5797 5798 return getCouldNotCompute(); 5799 } 5800 5801 ScalarEvolution::ExitLimit 5802 ScalarEvolution::computeExitLimitFromCond(const Loop *L, 5803 Value *ExitCond, 5804 BasicBlock *TBB, 5805 BasicBlock *FBB, 5806 bool ControlsExit, 5807 bool AllowPredicates) { 5808 // Check if the controlling expression for this loop is an And or Or. 5809 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(ExitCond)) { 5810 if (BO->getOpcode() == Instruction::And) { 5811 // Recurse on the operands of the and. 5812 bool EitherMayExit = L->contains(TBB); 5813 ExitLimit EL0 = computeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB, 5814 ControlsExit && !EitherMayExit, 5815 AllowPredicates); 5816 ExitLimit EL1 = computeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB, 5817 ControlsExit && !EitherMayExit, 5818 AllowPredicates); 5819 const SCEV *BECount = getCouldNotCompute(); 5820 const SCEV *MaxBECount = getCouldNotCompute(); 5821 if (EitherMayExit) { 5822 // Both conditions must be true for the loop to continue executing. 5823 // Choose the less conservative count. 5824 if (EL0.Exact == getCouldNotCompute() || 5825 EL1.Exact == getCouldNotCompute()) 5826 BECount = getCouldNotCompute(); 5827 else 5828 BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact); 5829 if (EL0.Max == getCouldNotCompute()) 5830 MaxBECount = EL1.Max; 5831 else if (EL1.Max == getCouldNotCompute()) 5832 MaxBECount = EL0.Max; 5833 else 5834 MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max); 5835 } else { 5836 // Both conditions must be true at the same time for the loop to exit. 5837 // For now, be conservative. 5838 assert(L->contains(FBB) && "Loop block has no successor in loop!"); 5839 if (EL0.Max == EL1.Max) 5840 MaxBECount = EL0.Max; 5841 if (EL0.Exact == EL1.Exact) 5842 BECount = EL0.Exact; 5843 } 5844 5845 SCEVUnionPredicate NP; 5846 NP.add(&EL0.Pred); 5847 NP.add(&EL1.Pred); 5848 // There are cases (e.g. PR26207) where computeExitLimitFromCond is able 5849 // to be more aggressive when computing BECount than when computing 5850 // MaxBECount. In these cases it is possible for EL0.Exact and EL1.Exact 5851 // to match, but for EL0.Max and EL1.Max to not. 5852 if (isa<SCEVCouldNotCompute>(MaxBECount) && 5853 !isa<SCEVCouldNotCompute>(BECount)) 5854 MaxBECount = BECount; 5855 5856 return ExitLimit(BECount, MaxBECount, NP); 5857 } 5858 if (BO->getOpcode() == Instruction::Or) { 5859 // Recurse on the operands of the or. 5860 bool EitherMayExit = L->contains(FBB); 5861 ExitLimit EL0 = computeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB, 5862 ControlsExit && !EitherMayExit, 5863 AllowPredicates); 5864 ExitLimit EL1 = computeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB, 5865 ControlsExit && !EitherMayExit, 5866 AllowPredicates); 5867 const SCEV *BECount = getCouldNotCompute(); 5868 const SCEV *MaxBECount = getCouldNotCompute(); 5869 if (EitherMayExit) { 5870 // Both conditions must be false for the loop to continue executing. 5871 // Choose the less conservative count. 5872 if (EL0.Exact == getCouldNotCompute() || 5873 EL1.Exact == getCouldNotCompute()) 5874 BECount = getCouldNotCompute(); 5875 else 5876 BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact); 5877 if (EL0.Max == getCouldNotCompute()) 5878 MaxBECount = EL1.Max; 5879 else if (EL1.Max == getCouldNotCompute()) 5880 MaxBECount = EL0.Max; 5881 else 5882 MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max); 5883 } else { 5884 // Both conditions must be false at the same time for the loop to exit. 5885 // For now, be conservative. 5886 assert(L->contains(TBB) && "Loop block has no successor in loop!"); 5887 if (EL0.Max == EL1.Max) 5888 MaxBECount = EL0.Max; 5889 if (EL0.Exact == EL1.Exact) 5890 BECount = EL0.Exact; 5891 } 5892 5893 SCEVUnionPredicate NP; 5894 NP.add(&EL0.Pred); 5895 NP.add(&EL1.Pred); 5896 return ExitLimit(BECount, MaxBECount, NP); 5897 } 5898 } 5899 5900 // With an icmp, it may be feasible to compute an exact backedge-taken count. 5901 // Proceed to the next level to examine the icmp. 5902 if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(ExitCond)) { 5903 ExitLimit EL = 5904 computeExitLimitFromICmp(L, ExitCondICmp, TBB, FBB, ControlsExit); 5905 if (EL.hasFullInfo() || !AllowPredicates) 5906 return EL; 5907 5908 // Try again, but use SCEV predicates this time. 5909 return computeExitLimitFromICmp(L, ExitCondICmp, TBB, FBB, ControlsExit, 5910 /*AllowPredicates=*/true); 5911 } 5912 5913 // Check for a constant condition. These are normally stripped out by 5914 // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to 5915 // preserve the CFG and is temporarily leaving constant conditions 5916 // in place. 5917 if (ConstantInt *CI = dyn_cast<ConstantInt>(ExitCond)) { 5918 if (L->contains(FBB) == !CI->getZExtValue()) 5919 // The backedge is always taken. 5920 return getCouldNotCompute(); 5921 else 5922 // The backedge is never taken. 5923 return getZero(CI->getType()); 5924 } 5925 5926 // If it's not an integer or pointer comparison then compute it the hard way. 5927 return computeExitCountExhaustively(L, ExitCond, !L->contains(TBB)); 5928 } 5929 5930 ScalarEvolution::ExitLimit 5931 ScalarEvolution::computeExitLimitFromICmp(const Loop *L, 5932 ICmpInst *ExitCond, 5933 BasicBlock *TBB, 5934 BasicBlock *FBB, 5935 bool ControlsExit, 5936 bool AllowPredicates) { 5937 5938 // If the condition was exit on true, convert the condition to exit on false 5939 ICmpInst::Predicate Cond; 5940 if (!L->contains(FBB)) 5941 Cond = ExitCond->getPredicate(); 5942 else 5943 Cond = ExitCond->getInversePredicate(); 5944 5945 // Handle common loops like: for (X = "string"; *X; ++X) 5946 if (LoadInst *LI = dyn_cast<LoadInst>(ExitCond->getOperand(0))) 5947 if (Constant *RHS = dyn_cast<Constant>(ExitCond->getOperand(1))) { 5948 ExitLimit ItCnt = 5949 computeLoadConstantCompareExitLimit(LI, RHS, L, Cond); 5950 if (ItCnt.hasAnyInfo()) 5951 return ItCnt; 5952 } 5953 5954 const SCEV *LHS = getSCEV(ExitCond->getOperand(0)); 5955 const SCEV *RHS = getSCEV(ExitCond->getOperand(1)); 5956 5957 // Try to evaluate any dependencies out of the loop. 5958 LHS = getSCEVAtScope(LHS, L); 5959 RHS = getSCEVAtScope(RHS, L); 5960 5961 // At this point, we would like to compute how many iterations of the 5962 // loop the predicate will return true for these inputs. 5963 if (isLoopInvariant(LHS, L) && !isLoopInvariant(RHS, L)) { 5964 // If there is a loop-invariant, force it into the RHS. 5965 std::swap(LHS, RHS); 5966 Cond = ICmpInst::getSwappedPredicate(Cond); 5967 } 5968 5969 // Simplify the operands before analyzing them. 5970 (void)SimplifyICmpOperands(Cond, LHS, RHS); 5971 5972 // If we have a comparison of a chrec against a constant, try to use value 5973 // ranges to answer this query. 5974 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) 5975 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(LHS)) 5976 if (AddRec->getLoop() == L) { 5977 // Form the constant range. 5978 ConstantRange CompRange( 5979 ICmpInst::makeConstantRange(Cond, RHSC->getAPInt())); 5980 5981 const SCEV *Ret = AddRec->getNumIterationsInRange(CompRange, *this); 5982 if (!isa<SCEVCouldNotCompute>(Ret)) return Ret; 5983 } 5984 5985 switch (Cond) { 5986 case ICmpInst::ICMP_NE: { // while (X != Y) 5987 // Convert to: while (X-Y != 0) 5988 ExitLimit EL = howFarToZero(getMinusSCEV(LHS, RHS), L, ControlsExit, 5989 AllowPredicates); 5990 if (EL.hasAnyInfo()) return EL; 5991 break; 5992 } 5993 case ICmpInst::ICMP_EQ: { // while (X == Y) 5994 // Convert to: while (X-Y == 0) 5995 ExitLimit EL = howFarToNonZero(getMinusSCEV(LHS, RHS), L); 5996 if (EL.hasAnyInfo()) return EL; 5997 break; 5998 } 5999 case ICmpInst::ICMP_SLT: 6000 case ICmpInst::ICMP_ULT: { // while (X < Y) 6001 bool IsSigned = Cond == ICmpInst::ICMP_SLT; 6002 ExitLimit EL = howManyLessThans(LHS, RHS, L, IsSigned, ControlsExit, 6003 AllowPredicates); 6004 if (EL.hasAnyInfo()) return EL; 6005 break; 6006 } 6007 case ICmpInst::ICMP_SGT: 6008 case ICmpInst::ICMP_UGT: { // while (X > Y) 6009 bool IsSigned = Cond == ICmpInst::ICMP_SGT; 6010 ExitLimit EL = 6011 howManyGreaterThans(LHS, RHS, L, IsSigned, ControlsExit, 6012 AllowPredicates); 6013 if (EL.hasAnyInfo()) return EL; 6014 break; 6015 } 6016 default: 6017 break; 6018 } 6019 6020 auto *ExhaustiveCount = 6021 computeExitCountExhaustively(L, ExitCond, !L->contains(TBB)); 6022 6023 if (!isa<SCEVCouldNotCompute>(ExhaustiveCount)) 6024 return ExhaustiveCount; 6025 6026 return computeShiftCompareExitLimit(ExitCond->getOperand(0), 6027 ExitCond->getOperand(1), L, Cond); 6028 } 6029 6030 ScalarEvolution::ExitLimit 6031 ScalarEvolution::computeExitLimitFromSingleExitSwitch(const Loop *L, 6032 SwitchInst *Switch, 6033 BasicBlock *ExitingBlock, 6034 bool ControlsExit) { 6035 assert(!L->contains(ExitingBlock) && "Not an exiting block!"); 6036 6037 // Give up if the exit is the default dest of a switch. 6038 if (Switch->getDefaultDest() == ExitingBlock) 6039 return getCouldNotCompute(); 6040 6041 assert(L->contains(Switch->getDefaultDest()) && 6042 "Default case must not exit the loop!"); 6043 const SCEV *LHS = getSCEVAtScope(Switch->getCondition(), L); 6044 const SCEV *RHS = getConstant(Switch->findCaseDest(ExitingBlock)); 6045 6046 // while (X != Y) --> while (X-Y != 0) 6047 ExitLimit EL = howFarToZero(getMinusSCEV(LHS, RHS), L, ControlsExit); 6048 if (EL.hasAnyInfo()) 6049 return EL; 6050 6051 return getCouldNotCompute(); 6052 } 6053 6054 static ConstantInt * 6055 EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C, 6056 ScalarEvolution &SE) { 6057 const SCEV *InVal = SE.getConstant(C); 6058 const SCEV *Val = AddRec->evaluateAtIteration(InVal, SE); 6059 assert(isa<SCEVConstant>(Val) && 6060 "Evaluation of SCEV at constant didn't fold correctly?"); 6061 return cast<SCEVConstant>(Val)->getValue(); 6062 } 6063 6064 /// Given an exit condition of 'icmp op load X, cst', try to see if we can 6065 /// compute the backedge execution count. 6066 ScalarEvolution::ExitLimit 6067 ScalarEvolution::computeLoadConstantCompareExitLimit( 6068 LoadInst *LI, 6069 Constant *RHS, 6070 const Loop *L, 6071 ICmpInst::Predicate predicate) { 6072 6073 if (LI->isVolatile()) return getCouldNotCompute(); 6074 6075 // Check to see if the loaded pointer is a getelementptr of a global. 6076 // TODO: Use SCEV instead of manually grubbing with GEPs. 6077 GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0)); 6078 if (!GEP) return getCouldNotCompute(); 6079 6080 // Make sure that it is really a constant global we are gepping, with an 6081 // initializer, and make sure the first IDX is really 0. 6082 GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)); 6083 if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer() || 6084 GEP->getNumOperands() < 3 || !isa<Constant>(GEP->getOperand(1)) || 6085 !cast<Constant>(GEP->getOperand(1))->isNullValue()) 6086 return getCouldNotCompute(); 6087 6088 // Okay, we allow one non-constant index into the GEP instruction. 6089 Value *VarIdx = nullptr; 6090 std::vector<Constant*> Indexes; 6091 unsigned VarIdxNum = 0; 6092 for (unsigned i = 2, e = GEP->getNumOperands(); i != e; ++i) 6093 if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) { 6094 Indexes.push_back(CI); 6095 } else if (!isa<ConstantInt>(GEP->getOperand(i))) { 6096 if (VarIdx) return getCouldNotCompute(); // Multiple non-constant idx's. 6097 VarIdx = GEP->getOperand(i); 6098 VarIdxNum = i-2; 6099 Indexes.push_back(nullptr); 6100 } 6101 6102 // Loop-invariant loads may be a byproduct of loop optimization. Skip them. 6103 if (!VarIdx) 6104 return getCouldNotCompute(); 6105 6106 // Okay, we know we have a (load (gep GV, 0, X)) comparison with a constant. 6107 // Check to see if X is a loop variant variable value now. 6108 const SCEV *Idx = getSCEV(VarIdx); 6109 Idx = getSCEVAtScope(Idx, L); 6110 6111 // We can only recognize very limited forms of loop index expressions, in 6112 // particular, only affine AddRec's like {C1,+,C2}. 6113 const SCEVAddRecExpr *IdxExpr = dyn_cast<SCEVAddRecExpr>(Idx); 6114 if (!IdxExpr || !IdxExpr->isAffine() || isLoopInvariant(IdxExpr, L) || 6115 !isa<SCEVConstant>(IdxExpr->getOperand(0)) || 6116 !isa<SCEVConstant>(IdxExpr->getOperand(1))) 6117 return getCouldNotCompute(); 6118 6119 unsigned MaxSteps = MaxBruteForceIterations; 6120 for (unsigned IterationNum = 0; IterationNum != MaxSteps; ++IterationNum) { 6121 ConstantInt *ItCst = ConstantInt::get( 6122 cast<IntegerType>(IdxExpr->getType()), IterationNum); 6123 ConstantInt *Val = EvaluateConstantChrecAtConstant(IdxExpr, ItCst, *this); 6124 6125 // Form the GEP offset. 6126 Indexes[VarIdxNum] = Val; 6127 6128 Constant *Result = ConstantFoldLoadThroughGEPIndices(GV->getInitializer(), 6129 Indexes); 6130 if (!Result) break; // Cannot compute! 6131 6132 // Evaluate the condition for this iteration. 6133 Result = ConstantExpr::getICmp(predicate, Result, RHS); 6134 if (!isa<ConstantInt>(Result)) break; // Couldn't decide for sure 6135 if (cast<ConstantInt>(Result)->getValue().isMinValue()) { 6136 ++NumArrayLenItCounts; 6137 return getConstant(ItCst); // Found terminating iteration! 6138 } 6139 } 6140 return getCouldNotCompute(); 6141 } 6142 6143 ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit( 6144 Value *LHS, Value *RHSV, const Loop *L, ICmpInst::Predicate Pred) { 6145 ConstantInt *RHS = dyn_cast<ConstantInt>(RHSV); 6146 if (!RHS) 6147 return getCouldNotCompute(); 6148 6149 const BasicBlock *Latch = L->getLoopLatch(); 6150 if (!Latch) 6151 return getCouldNotCompute(); 6152 6153 const BasicBlock *Predecessor = L->getLoopPredecessor(); 6154 if (!Predecessor) 6155 return getCouldNotCompute(); 6156 6157 // Return true if V is of the form "LHS `shift_op` <positive constant>". 6158 // Return LHS in OutLHS and shift_opt in OutOpCode. 6159 auto MatchPositiveShift = 6160 [](Value *V, Value *&OutLHS, Instruction::BinaryOps &OutOpCode) { 6161 6162 using namespace PatternMatch; 6163 6164 ConstantInt *ShiftAmt; 6165 if (match(V, m_LShr(m_Value(OutLHS), m_ConstantInt(ShiftAmt)))) 6166 OutOpCode = Instruction::LShr; 6167 else if (match(V, m_AShr(m_Value(OutLHS), m_ConstantInt(ShiftAmt)))) 6168 OutOpCode = Instruction::AShr; 6169 else if (match(V, m_Shl(m_Value(OutLHS), m_ConstantInt(ShiftAmt)))) 6170 OutOpCode = Instruction::Shl; 6171 else 6172 return false; 6173 6174 return ShiftAmt->getValue().isStrictlyPositive(); 6175 }; 6176 6177 // Recognize a "shift recurrence" either of the form %iv or of %iv.shifted in 6178 // 6179 // loop: 6180 // %iv = phi i32 [ %iv.shifted, %loop ], [ %val, %preheader ] 6181 // %iv.shifted = lshr i32 %iv, <positive constant> 6182 // 6183 // Return true on a succesful match. Return the corresponding PHI node (%iv 6184 // above) in PNOut and the opcode of the shift operation in OpCodeOut. 6185 auto MatchShiftRecurrence = 6186 [&](Value *V, PHINode *&PNOut, Instruction::BinaryOps &OpCodeOut) { 6187 Optional<Instruction::BinaryOps> PostShiftOpCode; 6188 6189 { 6190 Instruction::BinaryOps OpC; 6191 Value *V; 6192 6193 // If we encounter a shift instruction, "peel off" the shift operation, 6194 // and remember that we did so. Later when we inspect %iv's backedge 6195 // value, we will make sure that the backedge value uses the same 6196 // operation. 6197 // 6198 // Note: the peeled shift operation does not have to be the same 6199 // instruction as the one feeding into the PHI's backedge value. We only 6200 // really care about it being the same *kind* of shift instruction -- 6201 // that's all that is required for our later inferences to hold. 6202 if (MatchPositiveShift(LHS, V, OpC)) { 6203 PostShiftOpCode = OpC; 6204 LHS = V; 6205 } 6206 } 6207 6208 PNOut = dyn_cast<PHINode>(LHS); 6209 if (!PNOut || PNOut->getParent() != L->getHeader()) 6210 return false; 6211 6212 Value *BEValue = PNOut->getIncomingValueForBlock(Latch); 6213 Value *OpLHS; 6214 6215 return 6216 // The backedge value for the PHI node must be a shift by a positive 6217 // amount 6218 MatchPositiveShift(BEValue, OpLHS, OpCodeOut) && 6219 6220 // of the PHI node itself 6221 OpLHS == PNOut && 6222 6223 // and the kind of shift should be match the kind of shift we peeled 6224 // off, if any. 6225 (!PostShiftOpCode.hasValue() || *PostShiftOpCode == OpCodeOut); 6226 }; 6227 6228 PHINode *PN; 6229 Instruction::BinaryOps OpCode; 6230 if (!MatchShiftRecurrence(LHS, PN, OpCode)) 6231 return getCouldNotCompute(); 6232 6233 const DataLayout &DL = getDataLayout(); 6234 6235 // The key rationale for this optimization is that for some kinds of shift 6236 // recurrences, the value of the recurrence "stabilizes" to either 0 or -1 6237 // within a finite number of iterations. If the condition guarding the 6238 // backedge (in the sense that the backedge is taken if the condition is true) 6239 // is false for the value the shift recurrence stabilizes to, then we know 6240 // that the backedge is taken only a finite number of times. 6241 6242 ConstantInt *StableValue = nullptr; 6243 switch (OpCode) { 6244 default: 6245 llvm_unreachable("Impossible case!"); 6246 6247 case Instruction::AShr: { 6248 // {K,ashr,<positive-constant>} stabilizes to signum(K) in at most 6249 // bitwidth(K) iterations. 6250 Value *FirstValue = PN->getIncomingValueForBlock(Predecessor); 6251 bool KnownZero, KnownOne; 6252 ComputeSignBit(FirstValue, KnownZero, KnownOne, DL, 0, nullptr, 6253 Predecessor->getTerminator(), &DT); 6254 auto *Ty = cast<IntegerType>(RHS->getType()); 6255 if (KnownZero) 6256 StableValue = ConstantInt::get(Ty, 0); 6257 else if (KnownOne) 6258 StableValue = ConstantInt::get(Ty, -1, true); 6259 else 6260 return getCouldNotCompute(); 6261 6262 break; 6263 } 6264 case Instruction::LShr: 6265 case Instruction::Shl: 6266 // Both {K,lshr,<positive-constant>} and {K,shl,<positive-constant>} 6267 // stabilize to 0 in at most bitwidth(K) iterations. 6268 StableValue = ConstantInt::get(cast<IntegerType>(RHS->getType()), 0); 6269 break; 6270 } 6271 6272 auto *Result = 6273 ConstantFoldCompareInstOperands(Pred, StableValue, RHS, DL, &TLI); 6274 assert(Result->getType()->isIntegerTy(1) && 6275 "Otherwise cannot be an operand to a branch instruction"); 6276 6277 if (Result->isZeroValue()) { 6278 unsigned BitWidth = getTypeSizeInBits(RHS->getType()); 6279 const SCEV *UpperBound = 6280 getConstant(getEffectiveSCEVType(RHS->getType()), BitWidth); 6281 SCEVUnionPredicate P; 6282 return ExitLimit(getCouldNotCompute(), UpperBound, P); 6283 } 6284 6285 return getCouldNotCompute(); 6286 } 6287 6288 /// Return true if we can constant fold an instruction of the specified type, 6289 /// assuming that all operands were constants. 6290 static bool CanConstantFold(const Instruction *I) { 6291 if (isa<BinaryOperator>(I) || isa<CmpInst>(I) || 6292 isa<SelectInst>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) || 6293 isa<LoadInst>(I)) 6294 return true; 6295 6296 if (const CallInst *CI = dyn_cast<CallInst>(I)) 6297 if (const Function *F = CI->getCalledFunction()) 6298 return canConstantFoldCallTo(F); 6299 return false; 6300 } 6301 6302 /// Determine whether this instruction can constant evolve within this loop 6303 /// assuming its operands can all constant evolve. 6304 static bool canConstantEvolve(Instruction *I, const Loop *L) { 6305 // An instruction outside of the loop can't be derived from a loop PHI. 6306 if (!L->contains(I)) return false; 6307 6308 if (isa<PHINode>(I)) { 6309 // We don't currently keep track of the control flow needed to evaluate 6310 // PHIs, so we cannot handle PHIs inside of loops. 6311 return L->getHeader() == I->getParent(); 6312 } 6313 6314 // If we won't be able to constant fold this expression even if the operands 6315 // are constants, bail early. 6316 return CanConstantFold(I); 6317 } 6318 6319 /// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by 6320 /// recursing through each instruction operand until reaching a loop header phi. 6321 static PHINode * 6322 getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L, 6323 DenseMap<Instruction *, PHINode *> &PHIMap) { 6324 6325 // Otherwise, we can evaluate this instruction if all of its operands are 6326 // constant or derived from a PHI node themselves. 6327 PHINode *PHI = nullptr; 6328 for (Value *Op : UseInst->operands()) { 6329 if (isa<Constant>(Op)) continue; 6330 6331 Instruction *OpInst = dyn_cast<Instruction>(Op); 6332 if (!OpInst || !canConstantEvolve(OpInst, L)) return nullptr; 6333 6334 PHINode *P = dyn_cast<PHINode>(OpInst); 6335 if (!P) 6336 // If this operand is already visited, reuse the prior result. 6337 // We may have P != PHI if this is the deepest point at which the 6338 // inconsistent paths meet. 6339 P = PHIMap.lookup(OpInst); 6340 if (!P) { 6341 // Recurse and memoize the results, whether a phi is found or not. 6342 // This recursive call invalidates pointers into PHIMap. 6343 P = getConstantEvolvingPHIOperands(OpInst, L, PHIMap); 6344 PHIMap[OpInst] = P; 6345 } 6346 if (!P) 6347 return nullptr; // Not evolving from PHI 6348 if (PHI && PHI != P) 6349 return nullptr; // Evolving from multiple different PHIs. 6350 PHI = P; 6351 } 6352 // This is a expression evolving from a constant PHI! 6353 return PHI; 6354 } 6355 6356 /// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node 6357 /// in the loop that V is derived from. We allow arbitrary operations along the 6358 /// way, but the operands of an operation must either be constants or a value 6359 /// derived from a constant PHI. If this expression does not fit with these 6360 /// constraints, return null. 6361 static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) { 6362 Instruction *I = dyn_cast<Instruction>(V); 6363 if (!I || !canConstantEvolve(I, L)) return nullptr; 6364 6365 if (PHINode *PN = dyn_cast<PHINode>(I)) 6366 return PN; 6367 6368 // Record non-constant instructions contained by the loop. 6369 DenseMap<Instruction *, PHINode *> PHIMap; 6370 return getConstantEvolvingPHIOperands(I, L, PHIMap); 6371 } 6372 6373 /// EvaluateExpression - Given an expression that passes the 6374 /// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node 6375 /// in the loop has the value PHIVal. If we can't fold this expression for some 6376 /// reason, return null. 6377 static Constant *EvaluateExpression(Value *V, const Loop *L, 6378 DenseMap<Instruction *, Constant *> &Vals, 6379 const DataLayout &DL, 6380 const TargetLibraryInfo *TLI) { 6381 // Convenient constant check, but redundant for recursive calls. 6382 if (Constant *C = dyn_cast<Constant>(V)) return C; 6383 Instruction *I = dyn_cast<Instruction>(V); 6384 if (!I) return nullptr; 6385 6386 if (Constant *C = Vals.lookup(I)) return C; 6387 6388 // An instruction inside the loop depends on a value outside the loop that we 6389 // weren't given a mapping for, or a value such as a call inside the loop. 6390 if (!canConstantEvolve(I, L)) return nullptr; 6391 6392 // An unmapped PHI can be due to a branch or another loop inside this loop, 6393 // or due to this not being the initial iteration through a loop where we 6394 // couldn't compute the evolution of this particular PHI last time. 6395 if (isa<PHINode>(I)) return nullptr; 6396 6397 std::vector<Constant*> Operands(I->getNumOperands()); 6398 6399 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) { 6400 Instruction *Operand = dyn_cast<Instruction>(I->getOperand(i)); 6401 if (!Operand) { 6402 Operands[i] = dyn_cast<Constant>(I->getOperand(i)); 6403 if (!Operands[i]) return nullptr; 6404 continue; 6405 } 6406 Constant *C = EvaluateExpression(Operand, L, Vals, DL, TLI); 6407 Vals[Operand] = C; 6408 if (!C) return nullptr; 6409 Operands[i] = C; 6410 } 6411 6412 if (CmpInst *CI = dyn_cast<CmpInst>(I)) 6413 return ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0], 6414 Operands[1], DL, TLI); 6415 if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 6416 if (!LI->isVolatile()) 6417 return ConstantFoldLoadFromConstPtr(Operands[0], LI->getType(), DL); 6418 } 6419 return ConstantFoldInstOperands(I, Operands, DL, TLI); 6420 } 6421 6422 6423 // If every incoming value to PN except the one for BB is a specific Constant, 6424 // return that, else return nullptr. 6425 static Constant *getOtherIncomingValue(PHINode *PN, BasicBlock *BB) { 6426 Constant *IncomingVal = nullptr; 6427 6428 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { 6429 if (PN->getIncomingBlock(i) == BB) 6430 continue; 6431 6432 auto *CurrentVal = dyn_cast<Constant>(PN->getIncomingValue(i)); 6433 if (!CurrentVal) 6434 return nullptr; 6435 6436 if (IncomingVal != CurrentVal) { 6437 if (IncomingVal) 6438 return nullptr; 6439 IncomingVal = CurrentVal; 6440 } 6441 } 6442 6443 return IncomingVal; 6444 } 6445 6446 /// getConstantEvolutionLoopExitValue - If we know that the specified Phi is 6447 /// in the header of its containing loop, we know the loop executes a 6448 /// constant number of times, and the PHI node is just a recurrence 6449 /// involving constants, fold it. 6450 Constant * 6451 ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN, 6452 const APInt &BEs, 6453 const Loop *L) { 6454 auto I = ConstantEvolutionLoopExitValue.find(PN); 6455 if (I != ConstantEvolutionLoopExitValue.end()) 6456 return I->second; 6457 6458 if (BEs.ugt(MaxBruteForceIterations)) 6459 return ConstantEvolutionLoopExitValue[PN] = nullptr; // Not going to evaluate it. 6460 6461 Constant *&RetVal = ConstantEvolutionLoopExitValue[PN]; 6462 6463 DenseMap<Instruction *, Constant *> CurrentIterVals; 6464 BasicBlock *Header = L->getHeader(); 6465 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!"); 6466 6467 BasicBlock *Latch = L->getLoopLatch(); 6468 if (!Latch) 6469 return nullptr; 6470 6471 for (auto &I : *Header) { 6472 PHINode *PHI = dyn_cast<PHINode>(&I); 6473 if (!PHI) break; 6474 auto *StartCST = getOtherIncomingValue(PHI, Latch); 6475 if (!StartCST) continue; 6476 CurrentIterVals[PHI] = StartCST; 6477 } 6478 if (!CurrentIterVals.count(PN)) 6479 return RetVal = nullptr; 6480 6481 Value *BEValue = PN->getIncomingValueForBlock(Latch); 6482 6483 // Execute the loop symbolically to determine the exit value. 6484 if (BEs.getActiveBits() >= 32) 6485 return RetVal = nullptr; // More than 2^32-1 iterations?? Not doing it! 6486 6487 unsigned NumIterations = BEs.getZExtValue(); // must be in range 6488 unsigned IterationNum = 0; 6489 const DataLayout &DL = getDataLayout(); 6490 for (; ; ++IterationNum) { 6491 if (IterationNum == NumIterations) 6492 return RetVal = CurrentIterVals[PN]; // Got exit value! 6493 6494 // Compute the value of the PHIs for the next iteration. 6495 // EvaluateExpression adds non-phi values to the CurrentIterVals map. 6496 DenseMap<Instruction *, Constant *> NextIterVals; 6497 Constant *NextPHI = 6498 EvaluateExpression(BEValue, L, CurrentIterVals, DL, &TLI); 6499 if (!NextPHI) 6500 return nullptr; // Couldn't evaluate! 6501 NextIterVals[PN] = NextPHI; 6502 6503 bool StoppedEvolving = NextPHI == CurrentIterVals[PN]; 6504 6505 // Also evaluate the other PHI nodes. However, we don't get to stop if we 6506 // cease to be able to evaluate one of them or if they stop evolving, 6507 // because that doesn't necessarily prevent us from computing PN. 6508 SmallVector<std::pair<PHINode *, Constant *>, 8> PHIsToCompute; 6509 for (const auto &I : CurrentIterVals) { 6510 PHINode *PHI = dyn_cast<PHINode>(I.first); 6511 if (!PHI || PHI == PN || PHI->getParent() != Header) continue; 6512 PHIsToCompute.emplace_back(PHI, I.second); 6513 } 6514 // We use two distinct loops because EvaluateExpression may invalidate any 6515 // iterators into CurrentIterVals. 6516 for (const auto &I : PHIsToCompute) { 6517 PHINode *PHI = I.first; 6518 Constant *&NextPHI = NextIterVals[PHI]; 6519 if (!NextPHI) { // Not already computed. 6520 Value *BEValue = PHI->getIncomingValueForBlock(Latch); 6521 NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, &TLI); 6522 } 6523 if (NextPHI != I.second) 6524 StoppedEvolving = false; 6525 } 6526 6527 // If all entries in CurrentIterVals == NextIterVals then we can stop 6528 // iterating, the loop can't continue to change. 6529 if (StoppedEvolving) 6530 return RetVal = CurrentIterVals[PN]; 6531 6532 CurrentIterVals.swap(NextIterVals); 6533 } 6534 } 6535 6536 const SCEV *ScalarEvolution::computeExitCountExhaustively(const Loop *L, 6537 Value *Cond, 6538 bool ExitWhen) { 6539 PHINode *PN = getConstantEvolvingPHI(Cond, L); 6540 if (!PN) return getCouldNotCompute(); 6541 6542 // If the loop is canonicalized, the PHI will have exactly two entries. 6543 // That's the only form we support here. 6544 if (PN->getNumIncomingValues() != 2) return getCouldNotCompute(); 6545 6546 DenseMap<Instruction *, Constant *> CurrentIterVals; 6547 BasicBlock *Header = L->getHeader(); 6548 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!"); 6549 6550 BasicBlock *Latch = L->getLoopLatch(); 6551 assert(Latch && "Should follow from NumIncomingValues == 2!"); 6552 6553 for (auto &I : *Header) { 6554 PHINode *PHI = dyn_cast<PHINode>(&I); 6555 if (!PHI) 6556 break; 6557 auto *StartCST = getOtherIncomingValue(PHI, Latch); 6558 if (!StartCST) continue; 6559 CurrentIterVals[PHI] = StartCST; 6560 } 6561 if (!CurrentIterVals.count(PN)) 6562 return getCouldNotCompute(); 6563 6564 // Okay, we find a PHI node that defines the trip count of this loop. Execute 6565 // the loop symbolically to determine when the condition gets a value of 6566 // "ExitWhen". 6567 unsigned MaxIterations = MaxBruteForceIterations; // Limit analysis. 6568 const DataLayout &DL = getDataLayout(); 6569 for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){ 6570 auto *CondVal = dyn_cast_or_null<ConstantInt>( 6571 EvaluateExpression(Cond, L, CurrentIterVals, DL, &TLI)); 6572 6573 // Couldn't symbolically evaluate. 6574 if (!CondVal) return getCouldNotCompute(); 6575 6576 if (CondVal->getValue() == uint64_t(ExitWhen)) { 6577 ++NumBruteForceTripCountsComputed; 6578 return getConstant(Type::getInt32Ty(getContext()), IterationNum); 6579 } 6580 6581 // Update all the PHI nodes for the next iteration. 6582 DenseMap<Instruction *, Constant *> NextIterVals; 6583 6584 // Create a list of which PHIs we need to compute. We want to do this before 6585 // calling EvaluateExpression on them because that may invalidate iterators 6586 // into CurrentIterVals. 6587 SmallVector<PHINode *, 8> PHIsToCompute; 6588 for (const auto &I : CurrentIterVals) { 6589 PHINode *PHI = dyn_cast<PHINode>(I.first); 6590 if (!PHI || PHI->getParent() != Header) continue; 6591 PHIsToCompute.push_back(PHI); 6592 } 6593 for (PHINode *PHI : PHIsToCompute) { 6594 Constant *&NextPHI = NextIterVals[PHI]; 6595 if (NextPHI) continue; // Already computed! 6596 6597 Value *BEValue = PHI->getIncomingValueForBlock(Latch); 6598 NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, &TLI); 6599 } 6600 CurrentIterVals.swap(NextIterVals); 6601 } 6602 6603 // Too many iterations were needed to evaluate. 6604 return getCouldNotCompute(); 6605 } 6606 6607 const SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) { 6608 SmallVector<std::pair<const Loop *, const SCEV *>, 2> &Values = 6609 ValuesAtScopes[V]; 6610 // Check to see if we've folded this expression at this loop before. 6611 for (auto &LS : Values) 6612 if (LS.first == L) 6613 return LS.second ? LS.second : V; 6614 6615 Values.emplace_back(L, nullptr); 6616 6617 // Otherwise compute it. 6618 const SCEV *C = computeSCEVAtScope(V, L); 6619 for (auto &LS : reverse(ValuesAtScopes[V])) 6620 if (LS.first == L) { 6621 LS.second = C; 6622 break; 6623 } 6624 return C; 6625 } 6626 6627 /// This builds up a Constant using the ConstantExpr interface. That way, we 6628 /// will return Constants for objects which aren't represented by a 6629 /// SCEVConstant, because SCEVConstant is restricted to ConstantInt. 6630 /// Returns NULL if the SCEV isn't representable as a Constant. 6631 static Constant *BuildConstantFromSCEV(const SCEV *V) { 6632 switch (static_cast<SCEVTypes>(V->getSCEVType())) { 6633 case scCouldNotCompute: 6634 case scAddRecExpr: 6635 break; 6636 case scConstant: 6637 return cast<SCEVConstant>(V)->getValue(); 6638 case scUnknown: 6639 return dyn_cast<Constant>(cast<SCEVUnknown>(V)->getValue()); 6640 case scSignExtend: { 6641 const SCEVSignExtendExpr *SS = cast<SCEVSignExtendExpr>(V); 6642 if (Constant *CastOp = BuildConstantFromSCEV(SS->getOperand())) 6643 return ConstantExpr::getSExt(CastOp, SS->getType()); 6644 break; 6645 } 6646 case scZeroExtend: { 6647 const SCEVZeroExtendExpr *SZ = cast<SCEVZeroExtendExpr>(V); 6648 if (Constant *CastOp = BuildConstantFromSCEV(SZ->getOperand())) 6649 return ConstantExpr::getZExt(CastOp, SZ->getType()); 6650 break; 6651 } 6652 case scTruncate: { 6653 const SCEVTruncateExpr *ST = cast<SCEVTruncateExpr>(V); 6654 if (Constant *CastOp = BuildConstantFromSCEV(ST->getOperand())) 6655 return ConstantExpr::getTrunc(CastOp, ST->getType()); 6656 break; 6657 } 6658 case scAddExpr: { 6659 const SCEVAddExpr *SA = cast<SCEVAddExpr>(V); 6660 if (Constant *C = BuildConstantFromSCEV(SA->getOperand(0))) { 6661 if (PointerType *PTy = dyn_cast<PointerType>(C->getType())) { 6662 unsigned AS = PTy->getAddressSpace(); 6663 Type *DestPtrTy = Type::getInt8PtrTy(C->getContext(), AS); 6664 C = ConstantExpr::getBitCast(C, DestPtrTy); 6665 } 6666 for (unsigned i = 1, e = SA->getNumOperands(); i != e; ++i) { 6667 Constant *C2 = BuildConstantFromSCEV(SA->getOperand(i)); 6668 if (!C2) return nullptr; 6669 6670 // First pointer! 6671 if (!C->getType()->isPointerTy() && C2->getType()->isPointerTy()) { 6672 unsigned AS = C2->getType()->getPointerAddressSpace(); 6673 std::swap(C, C2); 6674 Type *DestPtrTy = Type::getInt8PtrTy(C->getContext(), AS); 6675 // The offsets have been converted to bytes. We can add bytes to an 6676 // i8* by GEP with the byte count in the first index. 6677 C = ConstantExpr::getBitCast(C, DestPtrTy); 6678 } 6679 6680 // Don't bother trying to sum two pointers. We probably can't 6681 // statically compute a load that results from it anyway. 6682 if (C2->getType()->isPointerTy()) 6683 return nullptr; 6684 6685 if (PointerType *PTy = dyn_cast<PointerType>(C->getType())) { 6686 if (PTy->getElementType()->isStructTy()) 6687 C2 = ConstantExpr::getIntegerCast( 6688 C2, Type::getInt32Ty(C->getContext()), true); 6689 C = ConstantExpr::getGetElementPtr(PTy->getElementType(), C, C2); 6690 } else 6691 C = ConstantExpr::getAdd(C, C2); 6692 } 6693 return C; 6694 } 6695 break; 6696 } 6697 case scMulExpr: { 6698 const SCEVMulExpr *SM = cast<SCEVMulExpr>(V); 6699 if (Constant *C = BuildConstantFromSCEV(SM->getOperand(0))) { 6700 // Don't bother with pointers at all. 6701 if (C->getType()->isPointerTy()) return nullptr; 6702 for (unsigned i = 1, e = SM->getNumOperands(); i != e; ++i) { 6703 Constant *C2 = BuildConstantFromSCEV(SM->getOperand(i)); 6704 if (!C2 || C2->getType()->isPointerTy()) return nullptr; 6705 C = ConstantExpr::getMul(C, C2); 6706 } 6707 return C; 6708 } 6709 break; 6710 } 6711 case scUDivExpr: { 6712 const SCEVUDivExpr *SU = cast<SCEVUDivExpr>(V); 6713 if (Constant *LHS = BuildConstantFromSCEV(SU->getLHS())) 6714 if (Constant *RHS = BuildConstantFromSCEV(SU->getRHS())) 6715 if (LHS->getType() == RHS->getType()) 6716 return ConstantExpr::getUDiv(LHS, RHS); 6717 break; 6718 } 6719 case scSMaxExpr: 6720 case scUMaxExpr: 6721 break; // TODO: smax, umax. 6722 } 6723 return nullptr; 6724 } 6725 6726 const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) { 6727 if (isa<SCEVConstant>(V)) return V; 6728 6729 // If this instruction is evolved from a constant-evolving PHI, compute the 6730 // exit value from the loop without using SCEVs. 6731 if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(V)) { 6732 if (Instruction *I = dyn_cast<Instruction>(SU->getValue())) { 6733 const Loop *LI = this->LI[I->getParent()]; 6734 if (LI && LI->getParentLoop() == L) // Looking for loop exit value. 6735 if (PHINode *PN = dyn_cast<PHINode>(I)) 6736 if (PN->getParent() == LI->getHeader()) { 6737 // Okay, there is no closed form solution for the PHI node. Check 6738 // to see if the loop that contains it has a known backedge-taken 6739 // count. If so, we may be able to force computation of the exit 6740 // value. 6741 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(LI); 6742 if (const SCEVConstant *BTCC = 6743 dyn_cast<SCEVConstant>(BackedgeTakenCount)) { 6744 // Okay, we know how many times the containing loop executes. If 6745 // this is a constant evolving PHI node, get the final value at 6746 // the specified iteration number. 6747 Constant *RV = 6748 getConstantEvolutionLoopExitValue(PN, BTCC->getAPInt(), LI); 6749 if (RV) return getSCEV(RV); 6750 } 6751 } 6752 6753 // Okay, this is an expression that we cannot symbolically evaluate 6754 // into a SCEV. Check to see if it's possible to symbolically evaluate 6755 // the arguments into constants, and if so, try to constant propagate the 6756 // result. This is particularly useful for computing loop exit values. 6757 if (CanConstantFold(I)) { 6758 SmallVector<Constant *, 4> Operands; 6759 bool MadeImprovement = false; 6760 for (Value *Op : I->operands()) { 6761 if (Constant *C = dyn_cast<Constant>(Op)) { 6762 Operands.push_back(C); 6763 continue; 6764 } 6765 6766 // If any of the operands is non-constant and if they are 6767 // non-integer and non-pointer, don't even try to analyze them 6768 // with scev techniques. 6769 if (!isSCEVable(Op->getType())) 6770 return V; 6771 6772 const SCEV *OrigV = getSCEV(Op); 6773 const SCEV *OpV = getSCEVAtScope(OrigV, L); 6774 MadeImprovement |= OrigV != OpV; 6775 6776 Constant *C = BuildConstantFromSCEV(OpV); 6777 if (!C) return V; 6778 if (C->getType() != Op->getType()) 6779 C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false, 6780 Op->getType(), 6781 false), 6782 C, Op->getType()); 6783 Operands.push_back(C); 6784 } 6785 6786 // Check to see if getSCEVAtScope actually made an improvement. 6787 if (MadeImprovement) { 6788 Constant *C = nullptr; 6789 const DataLayout &DL = getDataLayout(); 6790 if (const CmpInst *CI = dyn_cast<CmpInst>(I)) 6791 C = ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0], 6792 Operands[1], DL, &TLI); 6793 else if (const LoadInst *LI = dyn_cast<LoadInst>(I)) { 6794 if (!LI->isVolatile()) 6795 C = ConstantFoldLoadFromConstPtr(Operands[0], LI->getType(), DL); 6796 } else 6797 C = ConstantFoldInstOperands(I, Operands, DL, &TLI); 6798 if (!C) return V; 6799 return getSCEV(C); 6800 } 6801 } 6802 } 6803 6804 // This is some other type of SCEVUnknown, just return it. 6805 return V; 6806 } 6807 6808 if (const SCEVCommutativeExpr *Comm = dyn_cast<SCEVCommutativeExpr>(V)) { 6809 // Avoid performing the look-up in the common case where the specified 6810 // expression has no loop-variant portions. 6811 for (unsigned i = 0, e = Comm->getNumOperands(); i != e; ++i) { 6812 const SCEV *OpAtScope = getSCEVAtScope(Comm->getOperand(i), L); 6813 if (OpAtScope != Comm->getOperand(i)) { 6814 // Okay, at least one of these operands is loop variant but might be 6815 // foldable. Build a new instance of the folded commutative expression. 6816 SmallVector<const SCEV *, 8> NewOps(Comm->op_begin(), 6817 Comm->op_begin()+i); 6818 NewOps.push_back(OpAtScope); 6819 6820 for (++i; i != e; ++i) { 6821 OpAtScope = getSCEVAtScope(Comm->getOperand(i), L); 6822 NewOps.push_back(OpAtScope); 6823 } 6824 if (isa<SCEVAddExpr>(Comm)) 6825 return getAddExpr(NewOps); 6826 if (isa<SCEVMulExpr>(Comm)) 6827 return getMulExpr(NewOps); 6828 if (isa<SCEVSMaxExpr>(Comm)) 6829 return getSMaxExpr(NewOps); 6830 if (isa<SCEVUMaxExpr>(Comm)) 6831 return getUMaxExpr(NewOps); 6832 llvm_unreachable("Unknown commutative SCEV type!"); 6833 } 6834 } 6835 // If we got here, all operands are loop invariant. 6836 return Comm; 6837 } 6838 6839 if (const SCEVUDivExpr *Div = dyn_cast<SCEVUDivExpr>(V)) { 6840 const SCEV *LHS = getSCEVAtScope(Div->getLHS(), L); 6841 const SCEV *RHS = getSCEVAtScope(Div->getRHS(), L); 6842 if (LHS == Div->getLHS() && RHS == Div->getRHS()) 6843 return Div; // must be loop invariant 6844 return getUDivExpr(LHS, RHS); 6845 } 6846 6847 // If this is a loop recurrence for a loop that does not contain L, then we 6848 // are dealing with the final value computed by the loop. 6849 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V)) { 6850 // First, attempt to evaluate each operand. 6851 // Avoid performing the look-up in the common case where the specified 6852 // expression has no loop-variant portions. 6853 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) { 6854 const SCEV *OpAtScope = getSCEVAtScope(AddRec->getOperand(i), L); 6855 if (OpAtScope == AddRec->getOperand(i)) 6856 continue; 6857 6858 // Okay, at least one of these operands is loop variant but might be 6859 // foldable. Build a new instance of the folded commutative expression. 6860 SmallVector<const SCEV *, 8> NewOps(AddRec->op_begin(), 6861 AddRec->op_begin()+i); 6862 NewOps.push_back(OpAtScope); 6863 for (++i; i != e; ++i) 6864 NewOps.push_back(getSCEVAtScope(AddRec->getOperand(i), L)); 6865 6866 const SCEV *FoldedRec = 6867 getAddRecExpr(NewOps, AddRec->getLoop(), 6868 AddRec->getNoWrapFlags(SCEV::FlagNW)); 6869 AddRec = dyn_cast<SCEVAddRecExpr>(FoldedRec); 6870 // The addrec may be folded to a nonrecurrence, for example, if the 6871 // induction variable is multiplied by zero after constant folding. Go 6872 // ahead and return the folded value. 6873 if (!AddRec) 6874 return FoldedRec; 6875 break; 6876 } 6877 6878 // If the scope is outside the addrec's loop, evaluate it by using the 6879 // loop exit value of the addrec. 6880 if (!AddRec->getLoop()->contains(L)) { 6881 // To evaluate this recurrence, we need to know how many times the AddRec 6882 // loop iterates. Compute this now. 6883 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(AddRec->getLoop()); 6884 if (BackedgeTakenCount == getCouldNotCompute()) return AddRec; 6885 6886 // Then, evaluate the AddRec. 6887 return AddRec->evaluateAtIteration(BackedgeTakenCount, *this); 6888 } 6889 6890 return AddRec; 6891 } 6892 6893 if (const SCEVZeroExtendExpr *Cast = dyn_cast<SCEVZeroExtendExpr>(V)) { 6894 const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L); 6895 if (Op == Cast->getOperand()) 6896 return Cast; // must be loop invariant 6897 return getZeroExtendExpr(Op, Cast->getType()); 6898 } 6899 6900 if (const SCEVSignExtendExpr *Cast = dyn_cast<SCEVSignExtendExpr>(V)) { 6901 const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L); 6902 if (Op == Cast->getOperand()) 6903 return Cast; // must be loop invariant 6904 return getSignExtendExpr(Op, Cast->getType()); 6905 } 6906 6907 if (const SCEVTruncateExpr *Cast = dyn_cast<SCEVTruncateExpr>(V)) { 6908 const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L); 6909 if (Op == Cast->getOperand()) 6910 return Cast; // must be loop invariant 6911 return getTruncateExpr(Op, Cast->getType()); 6912 } 6913 6914 llvm_unreachable("Unknown SCEV type!"); 6915 } 6916 6917 const SCEV *ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) { 6918 return getSCEVAtScope(getSCEV(V), L); 6919 } 6920 6921 /// Finds the minimum unsigned root of the following equation: 6922 /// 6923 /// A * X = B (mod N) 6924 /// 6925 /// where N = 2^BW and BW is the common bit width of A and B. The signedness of 6926 /// A and B isn't important. 6927 /// 6928 /// If the equation does not have a solution, SCEVCouldNotCompute is returned. 6929 static const SCEV *SolveLinEquationWithOverflow(const APInt &A, const APInt &B, 6930 ScalarEvolution &SE) { 6931 uint32_t BW = A.getBitWidth(); 6932 assert(BW == B.getBitWidth() && "Bit widths must be the same."); 6933 assert(A != 0 && "A must be non-zero."); 6934 6935 // 1. D = gcd(A, N) 6936 // 6937 // The gcd of A and N may have only one prime factor: 2. The number of 6938 // trailing zeros in A is its multiplicity 6939 uint32_t Mult2 = A.countTrailingZeros(); 6940 // D = 2^Mult2 6941 6942 // 2. Check if B is divisible by D. 6943 // 6944 // B is divisible by D if and only if the multiplicity of prime factor 2 for B 6945 // is not less than multiplicity of this prime factor for D. 6946 if (B.countTrailingZeros() < Mult2) 6947 return SE.getCouldNotCompute(); 6948 6949 // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic 6950 // modulo (N / D). 6951 // 6952 // (N / D) may need BW+1 bits in its representation. Hence, we'll use this 6953 // bit width during computations. 6954 APInt AD = A.lshr(Mult2).zext(BW + 1); // AD = A / D 6955 APInt Mod(BW + 1, 0); 6956 Mod.setBit(BW - Mult2); // Mod = N / D 6957 APInt I = AD.multiplicativeInverse(Mod); 6958 6959 // 4. Compute the minimum unsigned root of the equation: 6960 // I * (B / D) mod (N / D) 6961 APInt Result = (I * B.lshr(Mult2).zext(BW + 1)).urem(Mod); 6962 6963 // The result is guaranteed to be less than 2^BW so we may truncate it to BW 6964 // bits. 6965 return SE.getConstant(Result.trunc(BW)); 6966 } 6967 6968 /// Find the roots of the quadratic equation for the given quadratic chrec 6969 /// {L,+,M,+,N}. This returns either the two roots (which might be the same) or 6970 /// two SCEVCouldNotCompute objects. 6971 /// 6972 static Optional<std::pair<const SCEVConstant *,const SCEVConstant *>> 6973 SolveQuadraticEquation(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) { 6974 assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!"); 6975 const SCEVConstant *LC = dyn_cast<SCEVConstant>(AddRec->getOperand(0)); 6976 const SCEVConstant *MC = dyn_cast<SCEVConstant>(AddRec->getOperand(1)); 6977 const SCEVConstant *NC = dyn_cast<SCEVConstant>(AddRec->getOperand(2)); 6978 6979 // We currently can only solve this if the coefficients are constants. 6980 if (!LC || !MC || !NC) 6981 return None; 6982 6983 uint32_t BitWidth = LC->getAPInt().getBitWidth(); 6984 const APInt &L = LC->getAPInt(); 6985 const APInt &M = MC->getAPInt(); 6986 const APInt &N = NC->getAPInt(); 6987 APInt Two(BitWidth, 2); 6988 APInt Four(BitWidth, 4); 6989 6990 { 6991 using namespace APIntOps; 6992 const APInt& C = L; 6993 // Convert from chrec coefficients to polynomial coefficients AX^2+BX+C 6994 // The B coefficient is M-N/2 6995 APInt B(M); 6996 B -= sdiv(N,Two); 6997 6998 // The A coefficient is N/2 6999 APInt A(N.sdiv(Two)); 7000 7001 // Compute the B^2-4ac term. 7002 APInt SqrtTerm(B); 7003 SqrtTerm *= B; 7004 SqrtTerm -= Four * (A * C); 7005 7006 if (SqrtTerm.isNegative()) { 7007 // The loop is provably infinite. 7008 return None; 7009 } 7010 7011 // Compute sqrt(B^2-4ac). This is guaranteed to be the nearest 7012 // integer value or else APInt::sqrt() will assert. 7013 APInt SqrtVal(SqrtTerm.sqrt()); 7014 7015 // Compute the two solutions for the quadratic formula. 7016 // The divisions must be performed as signed divisions. 7017 APInt NegB(-B); 7018 APInt TwoA(A << 1); 7019 if (TwoA.isMinValue()) 7020 return None; 7021 7022 LLVMContext &Context = SE.getContext(); 7023 7024 ConstantInt *Solution1 = 7025 ConstantInt::get(Context, (NegB + SqrtVal).sdiv(TwoA)); 7026 ConstantInt *Solution2 = 7027 ConstantInt::get(Context, (NegB - SqrtVal).sdiv(TwoA)); 7028 7029 return std::make_pair(cast<SCEVConstant>(SE.getConstant(Solution1)), 7030 cast<SCEVConstant>(SE.getConstant(Solution2))); 7031 } // end APIntOps namespace 7032 } 7033 7034 ScalarEvolution::ExitLimit 7035 ScalarEvolution::howFarToZero(const SCEV *V, const Loop *L, bool ControlsExit, 7036 bool AllowPredicates) { 7037 7038 // This is only used for loops with a "x != y" exit test. The exit condition 7039 // is now expressed as a single expression, V = x-y. So the exit test is 7040 // effectively V != 0. We know and take advantage of the fact that this 7041 // expression only being used in a comparison by zero context. 7042 7043 SCEVUnionPredicate P; 7044 // If the value is a constant 7045 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) { 7046 // If the value is already zero, the branch will execute zero times. 7047 if (C->getValue()->isZero()) return C; 7048 return getCouldNotCompute(); // Otherwise it will loop infinitely. 7049 } 7050 7051 const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V); 7052 if (!AddRec && AllowPredicates) 7053 // Try to make this an AddRec using runtime tests, in the first X 7054 // iterations of this loop, where X is the SCEV expression found by the 7055 // algorithm below. 7056 AddRec = convertSCEVToAddRecWithPredicates(V, L, P); 7057 7058 if (!AddRec || AddRec->getLoop() != L) 7059 return getCouldNotCompute(); 7060 7061 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of 7062 // the quadratic equation to solve it. 7063 if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) { 7064 if (auto Roots = SolveQuadraticEquation(AddRec, *this)) { 7065 const SCEVConstant *R1 = Roots->first; 7066 const SCEVConstant *R2 = Roots->second; 7067 // Pick the smallest positive root value. 7068 if (ConstantInt *CB = dyn_cast<ConstantInt>(ConstantExpr::getICmp( 7069 CmpInst::ICMP_ULT, R1->getValue(), R2->getValue()))) { 7070 if (!CB->getZExtValue()) 7071 std::swap(R1, R2); // R1 is the minimum root now. 7072 7073 // We can only use this value if the chrec ends up with an exact zero 7074 // value at this index. When solving for "X*X != 5", for example, we 7075 // should not accept a root of 2. 7076 const SCEV *Val = AddRec->evaluateAtIteration(R1, *this); 7077 if (Val->isZero()) 7078 return ExitLimit(R1, R1, P); // We found a quadratic root! 7079 } 7080 } 7081 return getCouldNotCompute(); 7082 } 7083 7084 // Otherwise we can only handle this if it is affine. 7085 if (!AddRec->isAffine()) 7086 return getCouldNotCompute(); 7087 7088 // If this is an affine expression, the execution count of this branch is 7089 // the minimum unsigned root of the following equation: 7090 // 7091 // Start + Step*N = 0 (mod 2^BW) 7092 // 7093 // equivalent to: 7094 // 7095 // Step*N = -Start (mod 2^BW) 7096 // 7097 // where BW is the common bit width of Start and Step. 7098 7099 // Get the initial value for the loop. 7100 const SCEV *Start = getSCEVAtScope(AddRec->getStart(), L->getParentLoop()); 7101 const SCEV *Step = getSCEVAtScope(AddRec->getOperand(1), L->getParentLoop()); 7102 7103 // For now we handle only constant steps. 7104 // 7105 // TODO: Handle a nonconstant Step given AddRec<NUW>. If the 7106 // AddRec is NUW, then (in an unsigned sense) it cannot be counting up to wrap 7107 // to 0, it must be counting down to equal 0. Consequently, N = Start / -Step. 7108 // We have not yet seen any such cases. 7109 const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Step); 7110 if (!StepC || StepC->getValue()->equalsInt(0)) 7111 return getCouldNotCompute(); 7112 7113 // For positive steps (counting up until unsigned overflow): 7114 // N = -Start/Step (as unsigned) 7115 // For negative steps (counting down to zero): 7116 // N = Start/-Step 7117 // First compute the unsigned distance from zero in the direction of Step. 7118 bool CountDown = StepC->getAPInt().isNegative(); 7119 const SCEV *Distance = CountDown ? Start : getNegativeSCEV(Start); 7120 7121 // Handle unitary steps, which cannot wraparound. 7122 // 1*N = -Start; -1*N = Start (mod 2^BW), so: 7123 // N = Distance (as unsigned) 7124 if (StepC->getValue()->equalsInt(1) || StepC->getValue()->isAllOnesValue()) { 7125 ConstantRange CR = getUnsignedRange(Start); 7126 const SCEV *MaxBECount; 7127 if (!CountDown && CR.getUnsignedMin().isMinValue()) 7128 // When counting up, the worst starting value is 1, not 0. 7129 MaxBECount = CR.getUnsignedMax().isMinValue() 7130 ? getConstant(APInt::getMinValue(CR.getBitWidth())) 7131 : getConstant(APInt::getMaxValue(CR.getBitWidth())); 7132 else 7133 MaxBECount = getConstant(CountDown ? CR.getUnsignedMax() 7134 : -CR.getUnsignedMin()); 7135 return ExitLimit(Distance, MaxBECount, P); 7136 } 7137 7138 // As a special case, handle the instance where Step is a positive power of 7139 // two. In this case, determining whether Step divides Distance evenly can be 7140 // done by counting and comparing the number of trailing zeros of Step and 7141 // Distance. 7142 if (!CountDown) { 7143 const APInt &StepV = StepC->getAPInt(); 7144 // StepV.isPowerOf2() returns true if StepV is an positive power of two. It 7145 // also returns true if StepV is maximally negative (eg, INT_MIN), but that 7146 // case is not handled as this code is guarded by !CountDown. 7147 if (StepV.isPowerOf2() && 7148 GetMinTrailingZeros(Distance) >= StepV.countTrailingZeros()) { 7149 // Here we've constrained the equation to be of the form 7150 // 7151 // 2^(N + k) * Distance' = (StepV == 2^N) * X (mod 2^W) ... (0) 7152 // 7153 // where we're operating on a W bit wide integer domain and k is 7154 // non-negative. The smallest unsigned solution for X is the trip count. 7155 // 7156 // (0) is equivalent to: 7157 // 7158 // 2^(N + k) * Distance' - 2^N * X = L * 2^W 7159 // <=> 2^N(2^k * Distance' - X) = L * 2^(W - N) * 2^N 7160 // <=> 2^k * Distance' - X = L * 2^(W - N) 7161 // <=> 2^k * Distance' = L * 2^(W - N) + X ... (1) 7162 // 7163 // The smallest X satisfying (1) is unsigned remainder of dividing the LHS 7164 // by 2^(W - N). 7165 // 7166 // <=> X = 2^k * Distance' URem 2^(W - N) ... (2) 7167 // 7168 // E.g. say we're solving 7169 // 7170 // 2 * Val = 2 * X (in i8) ... (3) 7171 // 7172 // then from (2), we get X = Val URem i8 128 (k = 0 in this case). 7173 // 7174 // Note: It is tempting to solve (3) by setting X = Val, but Val is not 7175 // necessarily the smallest unsigned value of X that satisfies (3). 7176 // E.g. if Val is i8 -127 then the smallest value of X that satisfies (3) 7177 // is i8 1, not i8 -127 7178 7179 const auto *ModuloResult = getUDivExactExpr(Distance, Step); 7180 7181 // Since SCEV does not have a URem node, we construct one using a truncate 7182 // and a zero extend. 7183 7184 unsigned NarrowWidth = StepV.getBitWidth() - StepV.countTrailingZeros(); 7185 auto *NarrowTy = IntegerType::get(getContext(), NarrowWidth); 7186 auto *WideTy = Distance->getType(); 7187 7188 const SCEV *Limit = 7189 getZeroExtendExpr(getTruncateExpr(ModuloResult, NarrowTy), WideTy); 7190 return ExitLimit(Limit, Limit, P); 7191 } 7192 } 7193 7194 // If the condition controls loop exit (the loop exits only if the expression 7195 // is true) and the addition is no-wrap we can use unsigned divide to 7196 // compute the backedge count. In this case, the step may not divide the 7197 // distance, but we don't care because if the condition is "missed" the loop 7198 // will have undefined behavior due to wrapping. 7199 if (ControlsExit && AddRec->hasNoSelfWrap() && 7200 loopHasNoAbnormalExits(AddRec->getLoop())) { 7201 const SCEV *Exact = 7202 getUDivExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step); 7203 return ExitLimit(Exact, Exact, P); 7204 } 7205 7206 // Then, try to solve the above equation provided that Start is constant. 7207 if (const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start)) { 7208 const SCEV *E = SolveLinEquationWithOverflow( 7209 StepC->getValue()->getValue(), -StartC->getValue()->getValue(), *this); 7210 return ExitLimit(E, E, P); 7211 } 7212 return getCouldNotCompute(); 7213 } 7214 7215 ScalarEvolution::ExitLimit 7216 ScalarEvolution::howFarToNonZero(const SCEV *V, const Loop *L) { 7217 // Loops that look like: while (X == 0) are very strange indeed. We don't 7218 // handle them yet except for the trivial case. This could be expanded in the 7219 // future as needed. 7220 7221 // If the value is a constant, check to see if it is known to be non-zero 7222 // already. If so, the backedge will execute zero times. 7223 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) { 7224 if (!C->getValue()->isNullValue()) 7225 return getZero(C->getType()); 7226 return getCouldNotCompute(); // Otherwise it will loop infinitely. 7227 } 7228 7229 // We could implement others, but I really doubt anyone writes loops like 7230 // this, and if they did, they would already be constant folded. 7231 return getCouldNotCompute(); 7232 } 7233 7234 std::pair<BasicBlock *, BasicBlock *> 7235 ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(BasicBlock *BB) { 7236 // If the block has a unique predecessor, then there is no path from the 7237 // predecessor to the block that does not go through the direct edge 7238 // from the predecessor to the block. 7239 if (BasicBlock *Pred = BB->getSinglePredecessor()) 7240 return {Pred, BB}; 7241 7242 // A loop's header is defined to be a block that dominates the loop. 7243 // If the header has a unique predecessor outside the loop, it must be 7244 // a block that has exactly one successor that can reach the loop. 7245 if (Loop *L = LI.getLoopFor(BB)) 7246 return {L->getLoopPredecessor(), L->getHeader()}; 7247 7248 return {nullptr, nullptr}; 7249 } 7250 7251 /// SCEV structural equivalence is usually sufficient for testing whether two 7252 /// expressions are equal, however for the purposes of looking for a condition 7253 /// guarding a loop, it can be useful to be a little more general, since a 7254 /// front-end may have replicated the controlling expression. 7255 /// 7256 static bool HasSameValue(const SCEV *A, const SCEV *B) { 7257 // Quick check to see if they are the same SCEV. 7258 if (A == B) return true; 7259 7260 auto ComputesEqualValues = [](const Instruction *A, const Instruction *B) { 7261 // Not all instructions that are "identical" compute the same value. For 7262 // instance, two distinct alloca instructions allocating the same type are 7263 // identical and do not read memory; but compute distinct values. 7264 return A->isIdenticalTo(B) && (isa<BinaryOperator>(A) || isa<GetElementPtrInst>(A)); 7265 }; 7266 7267 // Otherwise, if they're both SCEVUnknown, it's possible that they hold 7268 // two different instructions with the same value. Check for this case. 7269 if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(A)) 7270 if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(B)) 7271 if (const Instruction *AI = dyn_cast<Instruction>(AU->getValue())) 7272 if (const Instruction *BI = dyn_cast<Instruction>(BU->getValue())) 7273 if (ComputesEqualValues(AI, BI)) 7274 return true; 7275 7276 // Otherwise assume they may have a different value. 7277 return false; 7278 } 7279 7280 bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred, 7281 const SCEV *&LHS, const SCEV *&RHS, 7282 unsigned Depth) { 7283 bool Changed = false; 7284 7285 // If we hit the max recursion limit bail out. 7286 if (Depth >= 3) 7287 return false; 7288 7289 // Canonicalize a constant to the right side. 7290 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) { 7291 // Check for both operands constant. 7292 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) { 7293 if (ConstantExpr::getICmp(Pred, 7294 LHSC->getValue(), 7295 RHSC->getValue())->isNullValue()) 7296 goto trivially_false; 7297 else 7298 goto trivially_true; 7299 } 7300 // Otherwise swap the operands to put the constant on the right. 7301 std::swap(LHS, RHS); 7302 Pred = ICmpInst::getSwappedPredicate(Pred); 7303 Changed = true; 7304 } 7305 7306 // If we're comparing an addrec with a value which is loop-invariant in the 7307 // addrec's loop, put the addrec on the left. Also make a dominance check, 7308 // as both operands could be addrecs loop-invariant in each other's loop. 7309 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(RHS)) { 7310 const Loop *L = AR->getLoop(); 7311 if (isLoopInvariant(LHS, L) && properlyDominates(LHS, L->getHeader())) { 7312 std::swap(LHS, RHS); 7313 Pred = ICmpInst::getSwappedPredicate(Pred); 7314 Changed = true; 7315 } 7316 } 7317 7318 // If there's a constant operand, canonicalize comparisons with boundary 7319 // cases, and canonicalize *-or-equal comparisons to regular comparisons. 7320 if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS)) { 7321 const APInt &RA = RC->getAPInt(); 7322 switch (Pred) { 7323 default: llvm_unreachable("Unexpected ICmpInst::Predicate value!"); 7324 case ICmpInst::ICMP_EQ: 7325 case ICmpInst::ICMP_NE: 7326 // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b. 7327 if (!RA) 7328 if (const SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(LHS)) 7329 if (const SCEVMulExpr *ME = dyn_cast<SCEVMulExpr>(AE->getOperand(0))) 7330 if (AE->getNumOperands() == 2 && ME->getNumOperands() == 2 && 7331 ME->getOperand(0)->isAllOnesValue()) { 7332 RHS = AE->getOperand(1); 7333 LHS = ME->getOperand(1); 7334 Changed = true; 7335 } 7336 break; 7337 case ICmpInst::ICMP_UGE: 7338 if ((RA - 1).isMinValue()) { 7339 Pred = ICmpInst::ICMP_NE; 7340 RHS = getConstant(RA - 1); 7341 Changed = true; 7342 break; 7343 } 7344 if (RA.isMaxValue()) { 7345 Pred = ICmpInst::ICMP_EQ; 7346 Changed = true; 7347 break; 7348 } 7349 if (RA.isMinValue()) goto trivially_true; 7350 7351 Pred = ICmpInst::ICMP_UGT; 7352 RHS = getConstant(RA - 1); 7353 Changed = true; 7354 break; 7355 case ICmpInst::ICMP_ULE: 7356 if ((RA + 1).isMaxValue()) { 7357 Pred = ICmpInst::ICMP_NE; 7358 RHS = getConstant(RA + 1); 7359 Changed = true; 7360 break; 7361 } 7362 if (RA.isMinValue()) { 7363 Pred = ICmpInst::ICMP_EQ; 7364 Changed = true; 7365 break; 7366 } 7367 if (RA.isMaxValue()) goto trivially_true; 7368 7369 Pred = ICmpInst::ICMP_ULT; 7370 RHS = getConstant(RA + 1); 7371 Changed = true; 7372 break; 7373 case ICmpInst::ICMP_SGE: 7374 if ((RA - 1).isMinSignedValue()) { 7375 Pred = ICmpInst::ICMP_NE; 7376 RHS = getConstant(RA - 1); 7377 Changed = true; 7378 break; 7379 } 7380 if (RA.isMaxSignedValue()) { 7381 Pred = ICmpInst::ICMP_EQ; 7382 Changed = true; 7383 break; 7384 } 7385 if (RA.isMinSignedValue()) goto trivially_true; 7386 7387 Pred = ICmpInst::ICMP_SGT; 7388 RHS = getConstant(RA - 1); 7389 Changed = true; 7390 break; 7391 case ICmpInst::ICMP_SLE: 7392 if ((RA + 1).isMaxSignedValue()) { 7393 Pred = ICmpInst::ICMP_NE; 7394 RHS = getConstant(RA + 1); 7395 Changed = true; 7396 break; 7397 } 7398 if (RA.isMinSignedValue()) { 7399 Pred = ICmpInst::ICMP_EQ; 7400 Changed = true; 7401 break; 7402 } 7403 if (RA.isMaxSignedValue()) goto trivially_true; 7404 7405 Pred = ICmpInst::ICMP_SLT; 7406 RHS = getConstant(RA + 1); 7407 Changed = true; 7408 break; 7409 case ICmpInst::ICMP_UGT: 7410 if (RA.isMinValue()) { 7411 Pred = ICmpInst::ICMP_NE; 7412 Changed = true; 7413 break; 7414 } 7415 if ((RA + 1).isMaxValue()) { 7416 Pred = ICmpInst::ICMP_EQ; 7417 RHS = getConstant(RA + 1); 7418 Changed = true; 7419 break; 7420 } 7421 if (RA.isMaxValue()) goto trivially_false; 7422 break; 7423 case ICmpInst::ICMP_ULT: 7424 if (RA.isMaxValue()) { 7425 Pred = ICmpInst::ICMP_NE; 7426 Changed = true; 7427 break; 7428 } 7429 if ((RA - 1).isMinValue()) { 7430 Pred = ICmpInst::ICMP_EQ; 7431 RHS = getConstant(RA - 1); 7432 Changed = true; 7433 break; 7434 } 7435 if (RA.isMinValue()) goto trivially_false; 7436 break; 7437 case ICmpInst::ICMP_SGT: 7438 if (RA.isMinSignedValue()) { 7439 Pred = ICmpInst::ICMP_NE; 7440 Changed = true; 7441 break; 7442 } 7443 if ((RA + 1).isMaxSignedValue()) { 7444 Pred = ICmpInst::ICMP_EQ; 7445 RHS = getConstant(RA + 1); 7446 Changed = true; 7447 break; 7448 } 7449 if (RA.isMaxSignedValue()) goto trivially_false; 7450 break; 7451 case ICmpInst::ICMP_SLT: 7452 if (RA.isMaxSignedValue()) { 7453 Pred = ICmpInst::ICMP_NE; 7454 Changed = true; 7455 break; 7456 } 7457 if ((RA - 1).isMinSignedValue()) { 7458 Pred = ICmpInst::ICMP_EQ; 7459 RHS = getConstant(RA - 1); 7460 Changed = true; 7461 break; 7462 } 7463 if (RA.isMinSignedValue()) goto trivially_false; 7464 break; 7465 } 7466 } 7467 7468 // Check for obvious equality. 7469 if (HasSameValue(LHS, RHS)) { 7470 if (ICmpInst::isTrueWhenEqual(Pred)) 7471 goto trivially_true; 7472 if (ICmpInst::isFalseWhenEqual(Pred)) 7473 goto trivially_false; 7474 } 7475 7476 // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by 7477 // adding or subtracting 1 from one of the operands. 7478 switch (Pred) { 7479 case ICmpInst::ICMP_SLE: 7480 if (!getSignedRange(RHS).getSignedMax().isMaxSignedValue()) { 7481 RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS, 7482 SCEV::FlagNSW); 7483 Pred = ICmpInst::ICMP_SLT; 7484 Changed = true; 7485 } else if (!getSignedRange(LHS).getSignedMin().isMinSignedValue()) { 7486 LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS, 7487 SCEV::FlagNSW); 7488 Pred = ICmpInst::ICMP_SLT; 7489 Changed = true; 7490 } 7491 break; 7492 case ICmpInst::ICMP_SGE: 7493 if (!getSignedRange(RHS).getSignedMin().isMinSignedValue()) { 7494 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS, 7495 SCEV::FlagNSW); 7496 Pred = ICmpInst::ICMP_SGT; 7497 Changed = true; 7498 } else if (!getSignedRange(LHS).getSignedMax().isMaxSignedValue()) { 7499 LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS, 7500 SCEV::FlagNSW); 7501 Pred = ICmpInst::ICMP_SGT; 7502 Changed = true; 7503 } 7504 break; 7505 case ICmpInst::ICMP_ULE: 7506 if (!getUnsignedRange(RHS).getUnsignedMax().isMaxValue()) { 7507 RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS, 7508 SCEV::FlagNUW); 7509 Pred = ICmpInst::ICMP_ULT; 7510 Changed = true; 7511 } else if (!getUnsignedRange(LHS).getUnsignedMin().isMinValue()) { 7512 LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS); 7513 Pred = ICmpInst::ICMP_ULT; 7514 Changed = true; 7515 } 7516 break; 7517 case ICmpInst::ICMP_UGE: 7518 if (!getUnsignedRange(RHS).getUnsignedMin().isMinValue()) { 7519 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS); 7520 Pred = ICmpInst::ICMP_UGT; 7521 Changed = true; 7522 } else if (!getUnsignedRange(LHS).getUnsignedMax().isMaxValue()) { 7523 LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS, 7524 SCEV::FlagNUW); 7525 Pred = ICmpInst::ICMP_UGT; 7526 Changed = true; 7527 } 7528 break; 7529 default: 7530 break; 7531 } 7532 7533 // TODO: More simplifications are possible here. 7534 7535 // Recursively simplify until we either hit a recursion limit or nothing 7536 // changes. 7537 if (Changed) 7538 return SimplifyICmpOperands(Pred, LHS, RHS, Depth+1); 7539 7540 return Changed; 7541 7542 trivially_true: 7543 // Return 0 == 0. 7544 LHS = RHS = getConstant(ConstantInt::getFalse(getContext())); 7545 Pred = ICmpInst::ICMP_EQ; 7546 return true; 7547 7548 trivially_false: 7549 // Return 0 != 0. 7550 LHS = RHS = getConstant(ConstantInt::getFalse(getContext())); 7551 Pred = ICmpInst::ICMP_NE; 7552 return true; 7553 } 7554 7555 bool ScalarEvolution::isKnownNegative(const SCEV *S) { 7556 return getSignedRange(S).getSignedMax().isNegative(); 7557 } 7558 7559 bool ScalarEvolution::isKnownPositive(const SCEV *S) { 7560 return getSignedRange(S).getSignedMin().isStrictlyPositive(); 7561 } 7562 7563 bool ScalarEvolution::isKnownNonNegative(const SCEV *S) { 7564 return !getSignedRange(S).getSignedMin().isNegative(); 7565 } 7566 7567 bool ScalarEvolution::isKnownNonPositive(const SCEV *S) { 7568 return !getSignedRange(S).getSignedMax().isStrictlyPositive(); 7569 } 7570 7571 bool ScalarEvolution::isKnownNonZero(const SCEV *S) { 7572 return isKnownNegative(S) || isKnownPositive(S); 7573 } 7574 7575 bool ScalarEvolution::isKnownPredicate(ICmpInst::Predicate Pred, 7576 const SCEV *LHS, const SCEV *RHS) { 7577 // Canonicalize the inputs first. 7578 (void)SimplifyICmpOperands(Pred, LHS, RHS); 7579 7580 // If LHS or RHS is an addrec, check to see if the condition is true in 7581 // every iteration of the loop. 7582 // If LHS and RHS are both addrec, both conditions must be true in 7583 // every iteration of the loop. 7584 const SCEVAddRecExpr *LAR = dyn_cast<SCEVAddRecExpr>(LHS); 7585 const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(RHS); 7586 bool LeftGuarded = false; 7587 bool RightGuarded = false; 7588 if (LAR) { 7589 const Loop *L = LAR->getLoop(); 7590 if (isLoopEntryGuardedByCond(L, Pred, LAR->getStart(), RHS) && 7591 isLoopBackedgeGuardedByCond(L, Pred, LAR->getPostIncExpr(*this), RHS)) { 7592 if (!RAR) return true; 7593 LeftGuarded = true; 7594 } 7595 } 7596 if (RAR) { 7597 const Loop *L = RAR->getLoop(); 7598 if (isLoopEntryGuardedByCond(L, Pred, LHS, RAR->getStart()) && 7599 isLoopBackedgeGuardedByCond(L, Pred, LHS, RAR->getPostIncExpr(*this))) { 7600 if (!LAR) return true; 7601 RightGuarded = true; 7602 } 7603 } 7604 if (LeftGuarded && RightGuarded) 7605 return true; 7606 7607 if (isKnownPredicateViaSplitting(Pred, LHS, RHS)) 7608 return true; 7609 7610 // Otherwise see what can be done with known constant ranges. 7611 return isKnownPredicateViaConstantRanges(Pred, LHS, RHS); 7612 } 7613 7614 bool ScalarEvolution::isMonotonicPredicate(const SCEVAddRecExpr *LHS, 7615 ICmpInst::Predicate Pred, 7616 bool &Increasing) { 7617 bool Result = isMonotonicPredicateImpl(LHS, Pred, Increasing); 7618 7619 #ifndef NDEBUG 7620 // Verify an invariant: inverting the predicate should turn a monotonically 7621 // increasing change to a monotonically decreasing one, and vice versa. 7622 bool IncreasingSwapped; 7623 bool ResultSwapped = isMonotonicPredicateImpl( 7624 LHS, ICmpInst::getSwappedPredicate(Pred), IncreasingSwapped); 7625 7626 assert(Result == ResultSwapped && "should be able to analyze both!"); 7627 if (ResultSwapped) 7628 assert(Increasing == !IncreasingSwapped && 7629 "monotonicity should flip as we flip the predicate"); 7630 #endif 7631 7632 return Result; 7633 } 7634 7635 bool ScalarEvolution::isMonotonicPredicateImpl(const SCEVAddRecExpr *LHS, 7636 ICmpInst::Predicate Pred, 7637 bool &Increasing) { 7638 7639 // A zero step value for LHS means the induction variable is essentially a 7640 // loop invariant value. We don't really depend on the predicate actually 7641 // flipping from false to true (for increasing predicates, and the other way 7642 // around for decreasing predicates), all we care about is that *if* the 7643 // predicate changes then it only changes from false to true. 7644 // 7645 // A zero step value in itself is not very useful, but there may be places 7646 // where SCEV can prove X >= 0 but not prove X > 0, so it is helpful to be 7647 // as general as possible. 7648 7649 switch (Pred) { 7650 default: 7651 return false; // Conservative answer 7652 7653 case ICmpInst::ICMP_UGT: 7654 case ICmpInst::ICMP_UGE: 7655 case ICmpInst::ICMP_ULT: 7656 case ICmpInst::ICMP_ULE: 7657 if (!LHS->hasNoUnsignedWrap()) 7658 return false; 7659 7660 Increasing = Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE; 7661 return true; 7662 7663 case ICmpInst::ICMP_SGT: 7664 case ICmpInst::ICMP_SGE: 7665 case ICmpInst::ICMP_SLT: 7666 case ICmpInst::ICMP_SLE: { 7667 if (!LHS->hasNoSignedWrap()) 7668 return false; 7669 7670 const SCEV *Step = LHS->getStepRecurrence(*this); 7671 7672 if (isKnownNonNegative(Step)) { 7673 Increasing = Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE; 7674 return true; 7675 } 7676 7677 if (isKnownNonPositive(Step)) { 7678 Increasing = Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE; 7679 return true; 7680 } 7681 7682 return false; 7683 } 7684 7685 } 7686 7687 llvm_unreachable("switch has default clause!"); 7688 } 7689 7690 bool ScalarEvolution::isLoopInvariantPredicate( 7691 ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L, 7692 ICmpInst::Predicate &InvariantPred, const SCEV *&InvariantLHS, 7693 const SCEV *&InvariantRHS) { 7694 7695 // If there is a loop-invariant, force it into the RHS, otherwise bail out. 7696 if (!isLoopInvariant(RHS, L)) { 7697 if (!isLoopInvariant(LHS, L)) 7698 return false; 7699 7700 std::swap(LHS, RHS); 7701 Pred = ICmpInst::getSwappedPredicate(Pred); 7702 } 7703 7704 const SCEVAddRecExpr *ArLHS = dyn_cast<SCEVAddRecExpr>(LHS); 7705 if (!ArLHS || ArLHS->getLoop() != L) 7706 return false; 7707 7708 bool Increasing; 7709 if (!isMonotonicPredicate(ArLHS, Pred, Increasing)) 7710 return false; 7711 7712 // If the predicate "ArLHS `Pred` RHS" monotonically increases from false to 7713 // true as the loop iterates, and the backedge is control dependent on 7714 // "ArLHS `Pred` RHS" == true then we can reason as follows: 7715 // 7716 // * if the predicate was false in the first iteration then the predicate 7717 // is never evaluated again, since the loop exits without taking the 7718 // backedge. 7719 // * if the predicate was true in the first iteration then it will 7720 // continue to be true for all future iterations since it is 7721 // monotonically increasing. 7722 // 7723 // For both the above possibilities, we can replace the loop varying 7724 // predicate with its value on the first iteration of the loop (which is 7725 // loop invariant). 7726 // 7727 // A similar reasoning applies for a monotonically decreasing predicate, by 7728 // replacing true with false and false with true in the above two bullets. 7729 7730 auto P = Increasing ? Pred : ICmpInst::getInversePredicate(Pred); 7731 7732 if (!isLoopBackedgeGuardedByCond(L, P, LHS, RHS)) 7733 return false; 7734 7735 InvariantPred = Pred; 7736 InvariantLHS = ArLHS->getStart(); 7737 InvariantRHS = RHS; 7738 return true; 7739 } 7740 7741 bool ScalarEvolution::isKnownPredicateViaConstantRanges( 7742 ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS) { 7743 if (HasSameValue(LHS, RHS)) 7744 return ICmpInst::isTrueWhenEqual(Pred); 7745 7746 // This code is split out from isKnownPredicate because it is called from 7747 // within isLoopEntryGuardedByCond. 7748 7749 auto CheckRanges = 7750 [&](const ConstantRange &RangeLHS, const ConstantRange &RangeRHS) { 7751 return ConstantRange::makeSatisfyingICmpRegion(Pred, RangeRHS) 7752 .contains(RangeLHS); 7753 }; 7754 7755 // The check at the top of the function catches the case where the values are 7756 // known to be equal. 7757 if (Pred == CmpInst::ICMP_EQ) 7758 return false; 7759 7760 if (Pred == CmpInst::ICMP_NE) 7761 return CheckRanges(getSignedRange(LHS), getSignedRange(RHS)) || 7762 CheckRanges(getUnsignedRange(LHS), getUnsignedRange(RHS)) || 7763 isKnownNonZero(getMinusSCEV(LHS, RHS)); 7764 7765 if (CmpInst::isSigned(Pred)) 7766 return CheckRanges(getSignedRange(LHS), getSignedRange(RHS)); 7767 7768 return CheckRanges(getUnsignedRange(LHS), getUnsignedRange(RHS)); 7769 } 7770 7771 bool ScalarEvolution::isKnownPredicateViaNoOverflow(ICmpInst::Predicate Pred, 7772 const SCEV *LHS, 7773 const SCEV *RHS) { 7774 7775 // Match Result to (X + Y)<ExpectedFlags> where Y is a constant integer. 7776 // Return Y via OutY. 7777 auto MatchBinaryAddToConst = 7778 [this](const SCEV *Result, const SCEV *X, APInt &OutY, 7779 SCEV::NoWrapFlags ExpectedFlags) { 7780 const SCEV *NonConstOp, *ConstOp; 7781 SCEV::NoWrapFlags FlagsPresent; 7782 7783 if (!splitBinaryAdd(Result, ConstOp, NonConstOp, FlagsPresent) || 7784 !isa<SCEVConstant>(ConstOp) || NonConstOp != X) 7785 return false; 7786 7787 OutY = cast<SCEVConstant>(ConstOp)->getAPInt(); 7788 return (FlagsPresent & ExpectedFlags) == ExpectedFlags; 7789 }; 7790 7791 APInt C; 7792 7793 switch (Pred) { 7794 default: 7795 break; 7796 7797 case ICmpInst::ICMP_SGE: 7798 std::swap(LHS, RHS); 7799 case ICmpInst::ICMP_SLE: 7800 // X s<= (X + C)<nsw> if C >= 0 7801 if (MatchBinaryAddToConst(RHS, LHS, C, SCEV::FlagNSW) && C.isNonNegative()) 7802 return true; 7803 7804 // (X + C)<nsw> s<= X if C <= 0 7805 if (MatchBinaryAddToConst(LHS, RHS, C, SCEV::FlagNSW) && 7806 !C.isStrictlyPositive()) 7807 return true; 7808 break; 7809 7810 case ICmpInst::ICMP_SGT: 7811 std::swap(LHS, RHS); 7812 case ICmpInst::ICMP_SLT: 7813 // X s< (X + C)<nsw> if C > 0 7814 if (MatchBinaryAddToConst(RHS, LHS, C, SCEV::FlagNSW) && 7815 C.isStrictlyPositive()) 7816 return true; 7817 7818 // (X + C)<nsw> s< X if C < 0 7819 if (MatchBinaryAddToConst(LHS, RHS, C, SCEV::FlagNSW) && C.isNegative()) 7820 return true; 7821 break; 7822 } 7823 7824 return false; 7825 } 7826 7827 bool ScalarEvolution::isKnownPredicateViaSplitting(ICmpInst::Predicate Pred, 7828 const SCEV *LHS, 7829 const SCEV *RHS) { 7830 if (Pred != ICmpInst::ICMP_ULT || ProvingSplitPredicate) 7831 return false; 7832 7833 // Allowing arbitrary number of activations of isKnownPredicateViaSplitting on 7834 // the stack can result in exponential time complexity. 7835 SaveAndRestore<bool> Restore(ProvingSplitPredicate, true); 7836 7837 // If L >= 0 then I `ult` L <=> I >= 0 && I `slt` L 7838 // 7839 // To prove L >= 0 we use isKnownNonNegative whereas to prove I >= 0 we use 7840 // isKnownPredicate. isKnownPredicate is more powerful, but also more 7841 // expensive; and using isKnownNonNegative(RHS) is sufficient for most of the 7842 // interesting cases seen in practice. We can consider "upgrading" L >= 0 to 7843 // use isKnownPredicate later if needed. 7844 return isKnownNonNegative(RHS) && 7845 isKnownPredicate(CmpInst::ICMP_SGE, LHS, getZero(LHS->getType())) && 7846 isKnownPredicate(CmpInst::ICMP_SLT, LHS, RHS); 7847 } 7848 7849 bool ScalarEvolution::isImpliedViaGuard(BasicBlock *BB, 7850 ICmpInst::Predicate Pred, 7851 const SCEV *LHS, const SCEV *RHS) { 7852 // No need to even try if we know the module has no guards. 7853 if (!HasGuards) 7854 return false; 7855 7856 return any_of(*BB, [&](Instruction &I) { 7857 using namespace llvm::PatternMatch; 7858 7859 Value *Condition; 7860 return match(&I, m_Intrinsic<Intrinsic::experimental_guard>( 7861 m_Value(Condition))) && 7862 isImpliedCond(Pred, LHS, RHS, Condition, false); 7863 }); 7864 } 7865 7866 /// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is 7867 /// protected by a conditional between LHS and RHS. This is used to 7868 /// to eliminate casts. 7869 bool 7870 ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L, 7871 ICmpInst::Predicate Pred, 7872 const SCEV *LHS, const SCEV *RHS) { 7873 // Interpret a null as meaning no loop, where there is obviously no guard 7874 // (interprocedural conditions notwithstanding). 7875 if (!L) return true; 7876 7877 if (isKnownPredicateViaConstantRanges(Pred, LHS, RHS)) 7878 return true; 7879 7880 BasicBlock *Latch = L->getLoopLatch(); 7881 if (!Latch) 7882 return false; 7883 7884 BranchInst *LoopContinuePredicate = 7885 dyn_cast<BranchInst>(Latch->getTerminator()); 7886 if (LoopContinuePredicate && LoopContinuePredicate->isConditional() && 7887 isImpliedCond(Pred, LHS, RHS, 7888 LoopContinuePredicate->getCondition(), 7889 LoopContinuePredicate->getSuccessor(0) != L->getHeader())) 7890 return true; 7891 7892 // We don't want more than one activation of the following loops on the stack 7893 // -- that can lead to O(n!) time complexity. 7894 if (WalkingBEDominatingConds) 7895 return false; 7896 7897 SaveAndRestore<bool> ClearOnExit(WalkingBEDominatingConds, true); 7898 7899 // See if we can exploit a trip count to prove the predicate. 7900 const auto &BETakenInfo = getBackedgeTakenInfo(L); 7901 const SCEV *LatchBECount = BETakenInfo.getExact(Latch, this); 7902 if (LatchBECount != getCouldNotCompute()) { 7903 // We know that Latch branches back to the loop header exactly 7904 // LatchBECount times. This means the backdege condition at Latch is 7905 // equivalent to "{0,+,1} u< LatchBECount". 7906 Type *Ty = LatchBECount->getType(); 7907 auto NoWrapFlags = SCEV::NoWrapFlags(SCEV::FlagNUW | SCEV::FlagNW); 7908 const SCEV *LoopCounter = 7909 getAddRecExpr(getZero(Ty), getOne(Ty), L, NoWrapFlags); 7910 if (isImpliedCond(Pred, LHS, RHS, ICmpInst::ICMP_ULT, LoopCounter, 7911 LatchBECount)) 7912 return true; 7913 } 7914 7915 // Check conditions due to any @llvm.assume intrinsics. 7916 for (auto &AssumeVH : AC.assumptions()) { 7917 if (!AssumeVH) 7918 continue; 7919 auto *CI = cast<CallInst>(AssumeVH); 7920 if (!DT.dominates(CI, Latch->getTerminator())) 7921 continue; 7922 7923 if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false)) 7924 return true; 7925 } 7926 7927 // If the loop is not reachable from the entry block, we risk running into an 7928 // infinite loop as we walk up into the dom tree. These loops do not matter 7929 // anyway, so we just return a conservative answer when we see them. 7930 if (!DT.isReachableFromEntry(L->getHeader())) 7931 return false; 7932 7933 if (isImpliedViaGuard(Latch, Pred, LHS, RHS)) 7934 return true; 7935 7936 for (DomTreeNode *DTN = DT[Latch], *HeaderDTN = DT[L->getHeader()]; 7937 DTN != HeaderDTN; DTN = DTN->getIDom()) { 7938 7939 assert(DTN && "should reach the loop header before reaching the root!"); 7940 7941 BasicBlock *BB = DTN->getBlock(); 7942 if (isImpliedViaGuard(BB, Pred, LHS, RHS)) 7943 return true; 7944 7945 BasicBlock *PBB = BB->getSinglePredecessor(); 7946 if (!PBB) 7947 continue; 7948 7949 BranchInst *ContinuePredicate = dyn_cast<BranchInst>(PBB->getTerminator()); 7950 if (!ContinuePredicate || !ContinuePredicate->isConditional()) 7951 continue; 7952 7953 Value *Condition = ContinuePredicate->getCondition(); 7954 7955 // If we have an edge `E` within the loop body that dominates the only 7956 // latch, the condition guarding `E` also guards the backedge. This 7957 // reasoning works only for loops with a single latch. 7958 7959 BasicBlockEdge DominatingEdge(PBB, BB); 7960 if (DominatingEdge.isSingleEdge()) { 7961 // We're constructively (and conservatively) enumerating edges within the 7962 // loop body that dominate the latch. The dominator tree better agree 7963 // with us on this: 7964 assert(DT.dominates(DominatingEdge, Latch) && "should be!"); 7965 7966 if (isImpliedCond(Pred, LHS, RHS, Condition, 7967 BB != ContinuePredicate->getSuccessor(0))) 7968 return true; 7969 } 7970 } 7971 7972 return false; 7973 } 7974 7975 bool 7976 ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L, 7977 ICmpInst::Predicate Pred, 7978 const SCEV *LHS, const SCEV *RHS) { 7979 // Interpret a null as meaning no loop, where there is obviously no guard 7980 // (interprocedural conditions notwithstanding). 7981 if (!L) return false; 7982 7983 if (isKnownPredicateViaConstantRanges(Pred, LHS, RHS)) 7984 return true; 7985 7986 // Starting at the loop predecessor, climb up the predecessor chain, as long 7987 // as there are predecessors that can be found that have unique successors 7988 // leading to the original header. 7989 for (std::pair<BasicBlock *, BasicBlock *> 7990 Pair(L->getLoopPredecessor(), L->getHeader()); 7991 Pair.first; 7992 Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) { 7993 7994 if (isImpliedViaGuard(Pair.first, Pred, LHS, RHS)) 7995 return true; 7996 7997 BranchInst *LoopEntryPredicate = 7998 dyn_cast<BranchInst>(Pair.first->getTerminator()); 7999 if (!LoopEntryPredicate || 8000 LoopEntryPredicate->isUnconditional()) 8001 continue; 8002 8003 if (isImpliedCond(Pred, LHS, RHS, 8004 LoopEntryPredicate->getCondition(), 8005 LoopEntryPredicate->getSuccessor(0) != Pair.second)) 8006 return true; 8007 } 8008 8009 // Check conditions due to any @llvm.assume intrinsics. 8010 for (auto &AssumeVH : AC.assumptions()) { 8011 if (!AssumeVH) 8012 continue; 8013 auto *CI = cast<CallInst>(AssumeVH); 8014 if (!DT.dominates(CI, L->getHeader())) 8015 continue; 8016 8017 if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false)) 8018 return true; 8019 } 8020 8021 return false; 8022 } 8023 8024 namespace { 8025 /// RAII wrapper to prevent recursive application of isImpliedCond. 8026 /// ScalarEvolution's PendingLoopPredicates set must be empty unless we are 8027 /// currently evaluating isImpliedCond. 8028 struct MarkPendingLoopPredicate { 8029 Value *Cond; 8030 DenseSet<Value*> &LoopPreds; 8031 bool Pending; 8032 8033 MarkPendingLoopPredicate(Value *C, DenseSet<Value*> &LP) 8034 : Cond(C), LoopPreds(LP) { 8035 Pending = !LoopPreds.insert(Cond).second; 8036 } 8037 ~MarkPendingLoopPredicate() { 8038 if (!Pending) 8039 LoopPreds.erase(Cond); 8040 } 8041 }; 8042 } // end anonymous namespace 8043 8044 bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, 8045 const SCEV *LHS, const SCEV *RHS, 8046 Value *FoundCondValue, 8047 bool Inverse) { 8048 MarkPendingLoopPredicate Mark(FoundCondValue, PendingLoopPredicates); 8049 if (Mark.Pending) 8050 return false; 8051 8052 // Recursively handle And and Or conditions. 8053 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(FoundCondValue)) { 8054 if (BO->getOpcode() == Instruction::And) { 8055 if (!Inverse) 8056 return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) || 8057 isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse); 8058 } else if (BO->getOpcode() == Instruction::Or) { 8059 if (Inverse) 8060 return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) || 8061 isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse); 8062 } 8063 } 8064 8065 ICmpInst *ICI = dyn_cast<ICmpInst>(FoundCondValue); 8066 if (!ICI) return false; 8067 8068 // Now that we found a conditional branch that dominates the loop or controls 8069 // the loop latch. Check to see if it is the comparison we are looking for. 8070 ICmpInst::Predicate FoundPred; 8071 if (Inverse) 8072 FoundPred = ICI->getInversePredicate(); 8073 else 8074 FoundPred = ICI->getPredicate(); 8075 8076 const SCEV *FoundLHS = getSCEV(ICI->getOperand(0)); 8077 const SCEV *FoundRHS = getSCEV(ICI->getOperand(1)); 8078 8079 return isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS); 8080 } 8081 8082 bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, const SCEV *LHS, 8083 const SCEV *RHS, 8084 ICmpInst::Predicate FoundPred, 8085 const SCEV *FoundLHS, 8086 const SCEV *FoundRHS) { 8087 // Balance the types. 8088 if (getTypeSizeInBits(LHS->getType()) < 8089 getTypeSizeInBits(FoundLHS->getType())) { 8090 if (CmpInst::isSigned(Pred)) { 8091 LHS = getSignExtendExpr(LHS, FoundLHS->getType()); 8092 RHS = getSignExtendExpr(RHS, FoundLHS->getType()); 8093 } else { 8094 LHS = getZeroExtendExpr(LHS, FoundLHS->getType()); 8095 RHS = getZeroExtendExpr(RHS, FoundLHS->getType()); 8096 } 8097 } else if (getTypeSizeInBits(LHS->getType()) > 8098 getTypeSizeInBits(FoundLHS->getType())) { 8099 if (CmpInst::isSigned(FoundPred)) { 8100 FoundLHS = getSignExtendExpr(FoundLHS, LHS->getType()); 8101 FoundRHS = getSignExtendExpr(FoundRHS, LHS->getType()); 8102 } else { 8103 FoundLHS = getZeroExtendExpr(FoundLHS, LHS->getType()); 8104 FoundRHS = getZeroExtendExpr(FoundRHS, LHS->getType()); 8105 } 8106 } 8107 8108 // Canonicalize the query to match the way instcombine will have 8109 // canonicalized the comparison. 8110 if (SimplifyICmpOperands(Pred, LHS, RHS)) 8111 if (LHS == RHS) 8112 return CmpInst::isTrueWhenEqual(Pred); 8113 if (SimplifyICmpOperands(FoundPred, FoundLHS, FoundRHS)) 8114 if (FoundLHS == FoundRHS) 8115 return CmpInst::isFalseWhenEqual(FoundPred); 8116 8117 // Check to see if we can make the LHS or RHS match. 8118 if (LHS == FoundRHS || RHS == FoundLHS) { 8119 if (isa<SCEVConstant>(RHS)) { 8120 std::swap(FoundLHS, FoundRHS); 8121 FoundPred = ICmpInst::getSwappedPredicate(FoundPred); 8122 } else { 8123 std::swap(LHS, RHS); 8124 Pred = ICmpInst::getSwappedPredicate(Pred); 8125 } 8126 } 8127 8128 // Check whether the found predicate is the same as the desired predicate. 8129 if (FoundPred == Pred) 8130 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS); 8131 8132 // Check whether swapping the found predicate makes it the same as the 8133 // desired predicate. 8134 if (ICmpInst::getSwappedPredicate(FoundPred) == Pred) { 8135 if (isa<SCEVConstant>(RHS)) 8136 return isImpliedCondOperands(Pred, LHS, RHS, FoundRHS, FoundLHS); 8137 else 8138 return isImpliedCondOperands(ICmpInst::getSwappedPredicate(Pred), 8139 RHS, LHS, FoundLHS, FoundRHS); 8140 } 8141 8142 // Unsigned comparison is the same as signed comparison when both the operands 8143 // are non-negative. 8144 if (CmpInst::isUnsigned(FoundPred) && 8145 CmpInst::getSignedPredicate(FoundPred) == Pred && 8146 isKnownNonNegative(FoundLHS) && isKnownNonNegative(FoundRHS)) 8147 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS); 8148 8149 // Check if we can make progress by sharpening ranges. 8150 if (FoundPred == ICmpInst::ICMP_NE && 8151 (isa<SCEVConstant>(FoundLHS) || isa<SCEVConstant>(FoundRHS))) { 8152 8153 const SCEVConstant *C = nullptr; 8154 const SCEV *V = nullptr; 8155 8156 if (isa<SCEVConstant>(FoundLHS)) { 8157 C = cast<SCEVConstant>(FoundLHS); 8158 V = FoundRHS; 8159 } else { 8160 C = cast<SCEVConstant>(FoundRHS); 8161 V = FoundLHS; 8162 } 8163 8164 // The guarding predicate tells us that C != V. If the known range 8165 // of V is [C, t), we can sharpen the range to [C + 1, t). The 8166 // range we consider has to correspond to same signedness as the 8167 // predicate we're interested in folding. 8168 8169 APInt Min = ICmpInst::isSigned(Pred) ? 8170 getSignedRange(V).getSignedMin() : getUnsignedRange(V).getUnsignedMin(); 8171 8172 if (Min == C->getAPInt()) { 8173 // Given (V >= Min && V != Min) we conclude V >= (Min + 1). 8174 // This is true even if (Min + 1) wraps around -- in case of 8175 // wraparound, (Min + 1) < Min, so (V >= Min => V >= (Min + 1)). 8176 8177 APInt SharperMin = Min + 1; 8178 8179 switch (Pred) { 8180 case ICmpInst::ICMP_SGE: 8181 case ICmpInst::ICMP_UGE: 8182 // We know V `Pred` SharperMin. If this implies LHS `Pred` 8183 // RHS, we're done. 8184 if (isImpliedCondOperands(Pred, LHS, RHS, V, 8185 getConstant(SharperMin))) 8186 return true; 8187 8188 case ICmpInst::ICMP_SGT: 8189 case ICmpInst::ICMP_UGT: 8190 // We know from the range information that (V `Pred` Min || 8191 // V == Min). We know from the guarding condition that !(V 8192 // == Min). This gives us 8193 // 8194 // V `Pred` Min || V == Min && !(V == Min) 8195 // => V `Pred` Min 8196 // 8197 // If V `Pred` Min implies LHS `Pred` RHS, we're done. 8198 8199 if (isImpliedCondOperands(Pred, LHS, RHS, V, getConstant(Min))) 8200 return true; 8201 8202 default: 8203 // No change 8204 break; 8205 } 8206 } 8207 } 8208 8209 // Check whether the actual condition is beyond sufficient. 8210 if (FoundPred == ICmpInst::ICMP_EQ) 8211 if (ICmpInst::isTrueWhenEqual(Pred)) 8212 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS)) 8213 return true; 8214 if (Pred == ICmpInst::ICMP_NE) 8215 if (!ICmpInst::isTrueWhenEqual(FoundPred)) 8216 if (isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS)) 8217 return true; 8218 8219 // Otherwise assume the worst. 8220 return false; 8221 } 8222 8223 bool ScalarEvolution::splitBinaryAdd(const SCEV *Expr, 8224 const SCEV *&L, const SCEV *&R, 8225 SCEV::NoWrapFlags &Flags) { 8226 const auto *AE = dyn_cast<SCEVAddExpr>(Expr); 8227 if (!AE || AE->getNumOperands() != 2) 8228 return false; 8229 8230 L = AE->getOperand(0); 8231 R = AE->getOperand(1); 8232 Flags = AE->getNoWrapFlags(); 8233 return true; 8234 } 8235 8236 bool ScalarEvolution::computeConstantDifference(const SCEV *Less, 8237 const SCEV *More, 8238 APInt &C) { 8239 // We avoid subtracting expressions here because this function is usually 8240 // fairly deep in the call stack (i.e. is called many times). 8241 8242 if (isa<SCEVAddRecExpr>(Less) && isa<SCEVAddRecExpr>(More)) { 8243 const auto *LAR = cast<SCEVAddRecExpr>(Less); 8244 const auto *MAR = cast<SCEVAddRecExpr>(More); 8245 8246 if (LAR->getLoop() != MAR->getLoop()) 8247 return false; 8248 8249 // We look at affine expressions only; not for correctness but to keep 8250 // getStepRecurrence cheap. 8251 if (!LAR->isAffine() || !MAR->isAffine()) 8252 return false; 8253 8254 if (LAR->getStepRecurrence(*this) != MAR->getStepRecurrence(*this)) 8255 return false; 8256 8257 Less = LAR->getStart(); 8258 More = MAR->getStart(); 8259 8260 // fall through 8261 } 8262 8263 if (isa<SCEVConstant>(Less) && isa<SCEVConstant>(More)) { 8264 const auto &M = cast<SCEVConstant>(More)->getAPInt(); 8265 const auto &L = cast<SCEVConstant>(Less)->getAPInt(); 8266 C = M - L; 8267 return true; 8268 } 8269 8270 const SCEV *L, *R; 8271 SCEV::NoWrapFlags Flags; 8272 if (splitBinaryAdd(Less, L, R, Flags)) 8273 if (const auto *LC = dyn_cast<SCEVConstant>(L)) 8274 if (R == More) { 8275 C = -(LC->getAPInt()); 8276 return true; 8277 } 8278 8279 if (splitBinaryAdd(More, L, R, Flags)) 8280 if (const auto *LC = dyn_cast<SCEVConstant>(L)) 8281 if (R == Less) { 8282 C = LC->getAPInt(); 8283 return true; 8284 } 8285 8286 return false; 8287 } 8288 8289 bool ScalarEvolution::isImpliedCondOperandsViaNoOverflow( 8290 ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, 8291 const SCEV *FoundLHS, const SCEV *FoundRHS) { 8292 if (Pred != CmpInst::ICMP_SLT && Pred != CmpInst::ICMP_ULT) 8293 return false; 8294 8295 const auto *AddRecLHS = dyn_cast<SCEVAddRecExpr>(LHS); 8296 if (!AddRecLHS) 8297 return false; 8298 8299 const auto *AddRecFoundLHS = dyn_cast<SCEVAddRecExpr>(FoundLHS); 8300 if (!AddRecFoundLHS) 8301 return false; 8302 8303 // We'd like to let SCEV reason about control dependencies, so we constrain 8304 // both the inequalities to be about add recurrences on the same loop. This 8305 // way we can use isLoopEntryGuardedByCond later. 8306 8307 const Loop *L = AddRecFoundLHS->getLoop(); 8308 if (L != AddRecLHS->getLoop()) 8309 return false; 8310 8311 // FoundLHS u< FoundRHS u< -C => (FoundLHS + C) u< (FoundRHS + C) ... (1) 8312 // 8313 // FoundLHS s< FoundRHS s< INT_MIN - C => (FoundLHS + C) s< (FoundRHS + C) 8314 // ... (2) 8315 // 8316 // Informal proof for (2), assuming (1) [*]: 8317 // 8318 // We'll also assume (A s< B) <=> ((A + INT_MIN) u< (B + INT_MIN)) ... (3)[**] 8319 // 8320 // Then 8321 // 8322 // FoundLHS s< FoundRHS s< INT_MIN - C 8323 // <=> (FoundLHS + INT_MIN) u< (FoundRHS + INT_MIN) u< -C [ using (3) ] 8324 // <=> (FoundLHS + INT_MIN + C) u< (FoundRHS + INT_MIN + C) [ using (1) ] 8325 // <=> (FoundLHS + INT_MIN + C + INT_MIN) s< 8326 // (FoundRHS + INT_MIN + C + INT_MIN) [ using (3) ] 8327 // <=> FoundLHS + C s< FoundRHS + C 8328 // 8329 // [*]: (1) can be proved by ruling out overflow. 8330 // 8331 // [**]: This can be proved by analyzing all the four possibilities: 8332 // (A s< 0, B s< 0), (A s< 0, B s>= 0), (A s>= 0, B s< 0) and 8333 // (A s>= 0, B s>= 0). 8334 // 8335 // Note: 8336 // Despite (2), "FoundRHS s< INT_MIN - C" does not mean that "FoundRHS + C" 8337 // will not sign underflow. For instance, say FoundLHS = (i8 -128), FoundRHS 8338 // = (i8 -127) and C = (i8 -100). Then INT_MIN - C = (i8 -28), and FoundRHS 8339 // s< (INT_MIN - C). Lack of sign overflow / underflow in "FoundRHS + C" is 8340 // neither necessary nor sufficient to prove "(FoundLHS + C) s< (FoundRHS + 8341 // C)". 8342 8343 APInt LDiff, RDiff; 8344 if (!computeConstantDifference(FoundLHS, LHS, LDiff) || 8345 !computeConstantDifference(FoundRHS, RHS, RDiff) || 8346 LDiff != RDiff) 8347 return false; 8348 8349 if (LDiff == 0) 8350 return true; 8351 8352 APInt FoundRHSLimit; 8353 8354 if (Pred == CmpInst::ICMP_ULT) { 8355 FoundRHSLimit = -RDiff; 8356 } else { 8357 assert(Pred == CmpInst::ICMP_SLT && "Checked above!"); 8358 FoundRHSLimit = APInt::getSignedMinValue(getTypeSizeInBits(RHS->getType())) - RDiff; 8359 } 8360 8361 // Try to prove (1) or (2), as needed. 8362 return isLoopEntryGuardedByCond(L, Pred, FoundRHS, 8363 getConstant(FoundRHSLimit)); 8364 } 8365 8366 bool ScalarEvolution::isImpliedCondOperands(ICmpInst::Predicate Pred, 8367 const SCEV *LHS, const SCEV *RHS, 8368 const SCEV *FoundLHS, 8369 const SCEV *FoundRHS) { 8370 if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundLHS, FoundRHS)) 8371 return true; 8372 8373 if (isImpliedCondOperandsViaNoOverflow(Pred, LHS, RHS, FoundLHS, FoundRHS)) 8374 return true; 8375 8376 return isImpliedCondOperandsHelper(Pred, LHS, RHS, 8377 FoundLHS, FoundRHS) || 8378 // ~x < ~y --> x > y 8379 isImpliedCondOperandsHelper(Pred, LHS, RHS, 8380 getNotSCEV(FoundRHS), 8381 getNotSCEV(FoundLHS)); 8382 } 8383 8384 8385 /// If Expr computes ~A, return A else return nullptr 8386 static const SCEV *MatchNotExpr(const SCEV *Expr) { 8387 const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Expr); 8388 if (!Add || Add->getNumOperands() != 2 || 8389 !Add->getOperand(0)->isAllOnesValue()) 8390 return nullptr; 8391 8392 const SCEVMulExpr *AddRHS = dyn_cast<SCEVMulExpr>(Add->getOperand(1)); 8393 if (!AddRHS || AddRHS->getNumOperands() != 2 || 8394 !AddRHS->getOperand(0)->isAllOnesValue()) 8395 return nullptr; 8396 8397 return AddRHS->getOperand(1); 8398 } 8399 8400 8401 /// Is MaybeMaxExpr an SMax or UMax of Candidate and some other values? 8402 template<typename MaxExprType> 8403 static bool IsMaxConsistingOf(const SCEV *MaybeMaxExpr, 8404 const SCEV *Candidate) { 8405 const MaxExprType *MaxExpr = dyn_cast<MaxExprType>(MaybeMaxExpr); 8406 if (!MaxExpr) return false; 8407 8408 return find(MaxExpr->operands(), Candidate) != MaxExpr->op_end(); 8409 } 8410 8411 8412 /// Is MaybeMinExpr an SMin or UMin of Candidate and some other values? 8413 template<typename MaxExprType> 8414 static bool IsMinConsistingOf(ScalarEvolution &SE, 8415 const SCEV *MaybeMinExpr, 8416 const SCEV *Candidate) { 8417 const SCEV *MaybeMaxExpr = MatchNotExpr(MaybeMinExpr); 8418 if (!MaybeMaxExpr) 8419 return false; 8420 8421 return IsMaxConsistingOf<MaxExprType>(MaybeMaxExpr, SE.getNotSCEV(Candidate)); 8422 } 8423 8424 static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE, 8425 ICmpInst::Predicate Pred, 8426 const SCEV *LHS, const SCEV *RHS) { 8427 8428 // If both sides are affine addrecs for the same loop, with equal 8429 // steps, and we know the recurrences don't wrap, then we only 8430 // need to check the predicate on the starting values. 8431 8432 if (!ICmpInst::isRelational(Pred)) 8433 return false; 8434 8435 const SCEVAddRecExpr *LAR = dyn_cast<SCEVAddRecExpr>(LHS); 8436 if (!LAR) 8437 return false; 8438 const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(RHS); 8439 if (!RAR) 8440 return false; 8441 if (LAR->getLoop() != RAR->getLoop()) 8442 return false; 8443 if (!LAR->isAffine() || !RAR->isAffine()) 8444 return false; 8445 8446 if (LAR->getStepRecurrence(SE) != RAR->getStepRecurrence(SE)) 8447 return false; 8448 8449 SCEV::NoWrapFlags NW = ICmpInst::isSigned(Pred) ? 8450 SCEV::FlagNSW : SCEV::FlagNUW; 8451 if (!LAR->getNoWrapFlags(NW) || !RAR->getNoWrapFlags(NW)) 8452 return false; 8453 8454 return SE.isKnownPredicate(Pred, LAR->getStart(), RAR->getStart()); 8455 } 8456 8457 /// Is LHS `Pred` RHS true on the virtue of LHS or RHS being a Min or Max 8458 /// expression? 8459 static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE, 8460 ICmpInst::Predicate Pred, 8461 const SCEV *LHS, const SCEV *RHS) { 8462 switch (Pred) { 8463 default: 8464 return false; 8465 8466 case ICmpInst::ICMP_SGE: 8467 std::swap(LHS, RHS); 8468 // fall through 8469 case ICmpInst::ICMP_SLE: 8470 return 8471 // min(A, ...) <= A 8472 IsMinConsistingOf<SCEVSMaxExpr>(SE, LHS, RHS) || 8473 // A <= max(A, ...) 8474 IsMaxConsistingOf<SCEVSMaxExpr>(RHS, LHS); 8475 8476 case ICmpInst::ICMP_UGE: 8477 std::swap(LHS, RHS); 8478 // fall through 8479 case ICmpInst::ICMP_ULE: 8480 return 8481 // min(A, ...) <= A 8482 IsMinConsistingOf<SCEVUMaxExpr>(SE, LHS, RHS) || 8483 // A <= max(A, ...) 8484 IsMaxConsistingOf<SCEVUMaxExpr>(RHS, LHS); 8485 } 8486 8487 llvm_unreachable("covered switch fell through?!"); 8488 } 8489 8490 bool 8491 ScalarEvolution::isImpliedCondOperandsHelper(ICmpInst::Predicate Pred, 8492 const SCEV *LHS, const SCEV *RHS, 8493 const SCEV *FoundLHS, 8494 const SCEV *FoundRHS) { 8495 auto IsKnownPredicateFull = 8496 [this](ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS) { 8497 return isKnownPredicateViaConstantRanges(Pred, LHS, RHS) || 8498 IsKnownPredicateViaMinOrMax(*this, Pred, LHS, RHS) || 8499 IsKnownPredicateViaAddRecStart(*this, Pred, LHS, RHS) || 8500 isKnownPredicateViaNoOverflow(Pred, LHS, RHS); 8501 }; 8502 8503 switch (Pred) { 8504 default: llvm_unreachable("Unexpected ICmpInst::Predicate value!"); 8505 case ICmpInst::ICMP_EQ: 8506 case ICmpInst::ICMP_NE: 8507 if (HasSameValue(LHS, FoundLHS) && HasSameValue(RHS, FoundRHS)) 8508 return true; 8509 break; 8510 case ICmpInst::ICMP_SLT: 8511 case ICmpInst::ICMP_SLE: 8512 if (IsKnownPredicateFull(ICmpInst::ICMP_SLE, LHS, FoundLHS) && 8513 IsKnownPredicateFull(ICmpInst::ICMP_SGE, RHS, FoundRHS)) 8514 return true; 8515 break; 8516 case ICmpInst::ICMP_SGT: 8517 case ICmpInst::ICMP_SGE: 8518 if (IsKnownPredicateFull(ICmpInst::ICMP_SGE, LHS, FoundLHS) && 8519 IsKnownPredicateFull(ICmpInst::ICMP_SLE, RHS, FoundRHS)) 8520 return true; 8521 break; 8522 case ICmpInst::ICMP_ULT: 8523 case ICmpInst::ICMP_ULE: 8524 if (IsKnownPredicateFull(ICmpInst::ICMP_ULE, LHS, FoundLHS) && 8525 IsKnownPredicateFull(ICmpInst::ICMP_UGE, RHS, FoundRHS)) 8526 return true; 8527 break; 8528 case ICmpInst::ICMP_UGT: 8529 case ICmpInst::ICMP_UGE: 8530 if (IsKnownPredicateFull(ICmpInst::ICMP_UGE, LHS, FoundLHS) && 8531 IsKnownPredicateFull(ICmpInst::ICMP_ULE, RHS, FoundRHS)) 8532 return true; 8533 break; 8534 } 8535 8536 return false; 8537 } 8538 8539 bool ScalarEvolution::isImpliedCondOperandsViaRanges(ICmpInst::Predicate Pred, 8540 const SCEV *LHS, 8541 const SCEV *RHS, 8542 const SCEV *FoundLHS, 8543 const SCEV *FoundRHS) { 8544 if (!isa<SCEVConstant>(RHS) || !isa<SCEVConstant>(FoundRHS)) 8545 // The restriction on `FoundRHS` be lifted easily -- it exists only to 8546 // reduce the compile time impact of this optimization. 8547 return false; 8548 8549 const SCEVAddExpr *AddLHS = dyn_cast<SCEVAddExpr>(LHS); 8550 if (!AddLHS || AddLHS->getOperand(1) != FoundLHS || 8551 !isa<SCEVConstant>(AddLHS->getOperand(0))) 8552 return false; 8553 8554 APInt ConstFoundRHS = cast<SCEVConstant>(FoundRHS)->getAPInt(); 8555 8556 // `FoundLHSRange` is the range we know `FoundLHS` to be in by virtue of the 8557 // antecedent "`FoundLHS` `Pred` `FoundRHS`". 8558 ConstantRange FoundLHSRange = 8559 ConstantRange::makeAllowedICmpRegion(Pred, ConstFoundRHS); 8560 8561 // Since `LHS` is `FoundLHS` + `AddLHS->getOperand(0)`, we can compute a range 8562 // for `LHS`: 8563 APInt Addend = cast<SCEVConstant>(AddLHS->getOperand(0))->getAPInt(); 8564 ConstantRange LHSRange = FoundLHSRange.add(ConstantRange(Addend)); 8565 8566 // We can also compute the range of values for `LHS` that satisfy the 8567 // consequent, "`LHS` `Pred` `RHS`": 8568 APInt ConstRHS = cast<SCEVConstant>(RHS)->getAPInt(); 8569 ConstantRange SatisfyingLHSRange = 8570 ConstantRange::makeSatisfyingICmpRegion(Pred, ConstRHS); 8571 8572 // The antecedent implies the consequent if every value of `LHS` that 8573 // satisfies the antecedent also satisfies the consequent. 8574 return SatisfyingLHSRange.contains(LHSRange); 8575 } 8576 8577 bool ScalarEvolution::doesIVOverflowOnLT(const SCEV *RHS, const SCEV *Stride, 8578 bool IsSigned, bool NoWrap) { 8579 if (NoWrap) return false; 8580 8581 unsigned BitWidth = getTypeSizeInBits(RHS->getType()); 8582 const SCEV *One = getOne(Stride->getType()); 8583 8584 if (IsSigned) { 8585 APInt MaxRHS = getSignedRange(RHS).getSignedMax(); 8586 APInt MaxValue = APInt::getSignedMaxValue(BitWidth); 8587 APInt MaxStrideMinusOne = getSignedRange(getMinusSCEV(Stride, One)) 8588 .getSignedMax(); 8589 8590 // SMaxRHS + SMaxStrideMinusOne > SMaxValue => overflow! 8591 return (MaxValue - MaxStrideMinusOne).slt(MaxRHS); 8592 } 8593 8594 APInt MaxRHS = getUnsignedRange(RHS).getUnsignedMax(); 8595 APInt MaxValue = APInt::getMaxValue(BitWidth); 8596 APInt MaxStrideMinusOne = getUnsignedRange(getMinusSCEV(Stride, One)) 8597 .getUnsignedMax(); 8598 8599 // UMaxRHS + UMaxStrideMinusOne > UMaxValue => overflow! 8600 return (MaxValue - MaxStrideMinusOne).ult(MaxRHS); 8601 } 8602 8603 bool ScalarEvolution::doesIVOverflowOnGT(const SCEV *RHS, const SCEV *Stride, 8604 bool IsSigned, bool NoWrap) { 8605 if (NoWrap) return false; 8606 8607 unsigned BitWidth = getTypeSizeInBits(RHS->getType()); 8608 const SCEV *One = getOne(Stride->getType()); 8609 8610 if (IsSigned) { 8611 APInt MinRHS = getSignedRange(RHS).getSignedMin(); 8612 APInt MinValue = APInt::getSignedMinValue(BitWidth); 8613 APInt MaxStrideMinusOne = getSignedRange(getMinusSCEV(Stride, One)) 8614 .getSignedMax(); 8615 8616 // SMinRHS - SMaxStrideMinusOne < SMinValue => overflow! 8617 return (MinValue + MaxStrideMinusOne).sgt(MinRHS); 8618 } 8619 8620 APInt MinRHS = getUnsignedRange(RHS).getUnsignedMin(); 8621 APInt MinValue = APInt::getMinValue(BitWidth); 8622 APInt MaxStrideMinusOne = getUnsignedRange(getMinusSCEV(Stride, One)) 8623 .getUnsignedMax(); 8624 8625 // UMinRHS - UMaxStrideMinusOne < UMinValue => overflow! 8626 return (MinValue + MaxStrideMinusOne).ugt(MinRHS); 8627 } 8628 8629 const SCEV *ScalarEvolution::computeBECount(const SCEV *Delta, const SCEV *Step, 8630 bool Equality) { 8631 const SCEV *One = getOne(Step->getType()); 8632 Delta = Equality ? getAddExpr(Delta, Step) 8633 : getAddExpr(Delta, getMinusSCEV(Step, One)); 8634 return getUDivExpr(Delta, Step); 8635 } 8636 8637 ScalarEvolution::ExitLimit 8638 ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS, 8639 const Loop *L, bool IsSigned, 8640 bool ControlsExit, bool AllowPredicates) { 8641 SCEVUnionPredicate P; 8642 // We handle only IV < Invariant 8643 if (!isLoopInvariant(RHS, L)) 8644 return getCouldNotCompute(); 8645 8646 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(LHS); 8647 if (!IV && AllowPredicates) 8648 // Try to make this an AddRec using runtime tests, in the first X 8649 // iterations of this loop, where X is the SCEV expression found by the 8650 // algorithm below. 8651 IV = convertSCEVToAddRecWithPredicates(LHS, L, P); 8652 8653 // Avoid weird loops 8654 if (!IV || IV->getLoop() != L || !IV->isAffine()) 8655 return getCouldNotCompute(); 8656 8657 bool NoWrap = ControlsExit && 8658 IV->getNoWrapFlags(IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW); 8659 8660 const SCEV *Stride = IV->getStepRecurrence(*this); 8661 8662 // Avoid negative or zero stride values 8663 if (!isKnownPositive(Stride)) 8664 return getCouldNotCompute(); 8665 8666 // Avoid proven overflow cases: this will ensure that the backedge taken count 8667 // will not generate any unsigned overflow. Relaxed no-overflow conditions 8668 // exploit NoWrapFlags, allowing to optimize in presence of undefined 8669 // behaviors like the case of C language. 8670 if (!Stride->isOne() && doesIVOverflowOnLT(RHS, Stride, IsSigned, NoWrap)) 8671 return getCouldNotCompute(); 8672 8673 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SLT 8674 : ICmpInst::ICMP_ULT; 8675 const SCEV *Start = IV->getStart(); 8676 const SCEV *End = RHS; 8677 if (!isLoopEntryGuardedByCond(L, Cond, getMinusSCEV(Start, Stride), RHS)) 8678 End = IsSigned ? getSMaxExpr(RHS, Start) : getUMaxExpr(RHS, Start); 8679 8680 const SCEV *BECount = computeBECount(getMinusSCEV(End, Start), Stride, false); 8681 8682 APInt MinStart = IsSigned ? getSignedRange(Start).getSignedMin() 8683 : getUnsignedRange(Start).getUnsignedMin(); 8684 8685 APInt MinStride = IsSigned ? getSignedRange(Stride).getSignedMin() 8686 : getUnsignedRange(Stride).getUnsignedMin(); 8687 8688 unsigned BitWidth = getTypeSizeInBits(LHS->getType()); 8689 APInt Limit = IsSigned ? APInt::getSignedMaxValue(BitWidth) - (MinStride - 1) 8690 : APInt::getMaxValue(BitWidth) - (MinStride - 1); 8691 8692 // Although End can be a MAX expression we estimate MaxEnd considering only 8693 // the case End = RHS. This is safe because in the other case (End - Start) 8694 // is zero, leading to a zero maximum backedge taken count. 8695 APInt MaxEnd = 8696 IsSigned ? APIntOps::smin(getSignedRange(RHS).getSignedMax(), Limit) 8697 : APIntOps::umin(getUnsignedRange(RHS).getUnsignedMax(), Limit); 8698 8699 const SCEV *MaxBECount; 8700 if (isa<SCEVConstant>(BECount)) 8701 MaxBECount = BECount; 8702 else 8703 MaxBECount = computeBECount(getConstant(MaxEnd - MinStart), 8704 getConstant(MinStride), false); 8705 8706 if (isa<SCEVCouldNotCompute>(MaxBECount)) 8707 MaxBECount = BECount; 8708 8709 return ExitLimit(BECount, MaxBECount, P); 8710 } 8711 8712 ScalarEvolution::ExitLimit 8713 ScalarEvolution::howManyGreaterThans(const SCEV *LHS, const SCEV *RHS, 8714 const Loop *L, bool IsSigned, 8715 bool ControlsExit, bool AllowPredicates) { 8716 SCEVUnionPredicate P; 8717 // We handle only IV > Invariant 8718 if (!isLoopInvariant(RHS, L)) 8719 return getCouldNotCompute(); 8720 8721 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(LHS); 8722 if (!IV && AllowPredicates) 8723 // Try to make this an AddRec using runtime tests, in the first X 8724 // iterations of this loop, where X is the SCEV expression found by the 8725 // algorithm below. 8726 IV = convertSCEVToAddRecWithPredicates(LHS, L, P); 8727 8728 // Avoid weird loops 8729 if (!IV || IV->getLoop() != L || !IV->isAffine()) 8730 return getCouldNotCompute(); 8731 8732 bool NoWrap = ControlsExit && 8733 IV->getNoWrapFlags(IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW); 8734 8735 const SCEV *Stride = getNegativeSCEV(IV->getStepRecurrence(*this)); 8736 8737 // Avoid negative or zero stride values 8738 if (!isKnownPositive(Stride)) 8739 return getCouldNotCompute(); 8740 8741 // Avoid proven overflow cases: this will ensure that the backedge taken count 8742 // will not generate any unsigned overflow. Relaxed no-overflow conditions 8743 // exploit NoWrapFlags, allowing to optimize in presence of undefined 8744 // behaviors like the case of C language. 8745 if (!Stride->isOne() && doesIVOverflowOnGT(RHS, Stride, IsSigned, NoWrap)) 8746 return getCouldNotCompute(); 8747 8748 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SGT 8749 : ICmpInst::ICMP_UGT; 8750 8751 const SCEV *Start = IV->getStart(); 8752 const SCEV *End = RHS; 8753 if (!isLoopEntryGuardedByCond(L, Cond, getAddExpr(Start, Stride), RHS)) 8754 End = IsSigned ? getSMinExpr(RHS, Start) : getUMinExpr(RHS, Start); 8755 8756 const SCEV *BECount = computeBECount(getMinusSCEV(Start, End), Stride, false); 8757 8758 APInt MaxStart = IsSigned ? getSignedRange(Start).getSignedMax() 8759 : getUnsignedRange(Start).getUnsignedMax(); 8760 8761 APInt MinStride = IsSigned ? getSignedRange(Stride).getSignedMin() 8762 : getUnsignedRange(Stride).getUnsignedMin(); 8763 8764 unsigned BitWidth = getTypeSizeInBits(LHS->getType()); 8765 APInt Limit = IsSigned ? APInt::getSignedMinValue(BitWidth) + (MinStride - 1) 8766 : APInt::getMinValue(BitWidth) + (MinStride - 1); 8767 8768 // Although End can be a MIN expression we estimate MinEnd considering only 8769 // the case End = RHS. This is safe because in the other case (Start - End) 8770 // is zero, leading to a zero maximum backedge taken count. 8771 APInt MinEnd = 8772 IsSigned ? APIntOps::smax(getSignedRange(RHS).getSignedMin(), Limit) 8773 : APIntOps::umax(getUnsignedRange(RHS).getUnsignedMin(), Limit); 8774 8775 8776 const SCEV *MaxBECount = getCouldNotCompute(); 8777 if (isa<SCEVConstant>(BECount)) 8778 MaxBECount = BECount; 8779 else 8780 MaxBECount = computeBECount(getConstant(MaxStart - MinEnd), 8781 getConstant(MinStride), false); 8782 8783 if (isa<SCEVCouldNotCompute>(MaxBECount)) 8784 MaxBECount = BECount; 8785 8786 return ExitLimit(BECount, MaxBECount, P); 8787 } 8788 8789 const SCEV *SCEVAddRecExpr::getNumIterationsInRange(const ConstantRange &Range, 8790 ScalarEvolution &SE) const { 8791 if (Range.isFullSet()) // Infinite loop. 8792 return SE.getCouldNotCompute(); 8793 8794 // If the start is a non-zero constant, shift the range to simplify things. 8795 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(getStart())) 8796 if (!SC->getValue()->isZero()) { 8797 SmallVector<const SCEV *, 4> Operands(op_begin(), op_end()); 8798 Operands[0] = SE.getZero(SC->getType()); 8799 const SCEV *Shifted = SE.getAddRecExpr(Operands, getLoop(), 8800 getNoWrapFlags(FlagNW)); 8801 if (const auto *ShiftedAddRec = dyn_cast<SCEVAddRecExpr>(Shifted)) 8802 return ShiftedAddRec->getNumIterationsInRange( 8803 Range.subtract(SC->getAPInt()), SE); 8804 // This is strange and shouldn't happen. 8805 return SE.getCouldNotCompute(); 8806 } 8807 8808 // The only time we can solve this is when we have all constant indices. 8809 // Otherwise, we cannot determine the overflow conditions. 8810 if (any_of(operands(), [](const SCEV *Op) { return !isa<SCEVConstant>(Op); })) 8811 return SE.getCouldNotCompute(); 8812 8813 // Okay at this point we know that all elements of the chrec are constants and 8814 // that the start element is zero. 8815 8816 // First check to see if the range contains zero. If not, the first 8817 // iteration exits. 8818 unsigned BitWidth = SE.getTypeSizeInBits(getType()); 8819 if (!Range.contains(APInt(BitWidth, 0))) 8820 return SE.getZero(getType()); 8821 8822 if (isAffine()) { 8823 // If this is an affine expression then we have this situation: 8824 // Solve {0,+,A} in Range === Ax in Range 8825 8826 // We know that zero is in the range. If A is positive then we know that 8827 // the upper value of the range must be the first possible exit value. 8828 // If A is negative then the lower of the range is the last possible loop 8829 // value. Also note that we already checked for a full range. 8830 APInt One(BitWidth,1); 8831 APInt A = cast<SCEVConstant>(getOperand(1))->getAPInt(); 8832 APInt End = A.sge(One) ? (Range.getUpper() - One) : Range.getLower(); 8833 8834 // The exit value should be (End+A)/A. 8835 APInt ExitVal = (End + A).udiv(A); 8836 ConstantInt *ExitValue = ConstantInt::get(SE.getContext(), ExitVal); 8837 8838 // Evaluate at the exit value. If we really did fall out of the valid 8839 // range, then we computed our trip count, otherwise wrap around or other 8840 // things must have happened. 8841 ConstantInt *Val = EvaluateConstantChrecAtConstant(this, ExitValue, SE); 8842 if (Range.contains(Val->getValue())) 8843 return SE.getCouldNotCompute(); // Something strange happened 8844 8845 // Ensure that the previous value is in the range. This is a sanity check. 8846 assert(Range.contains( 8847 EvaluateConstantChrecAtConstant(this, 8848 ConstantInt::get(SE.getContext(), ExitVal - One), SE)->getValue()) && 8849 "Linear scev computation is off in a bad way!"); 8850 return SE.getConstant(ExitValue); 8851 } else if (isQuadratic()) { 8852 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of the 8853 // quadratic equation to solve it. To do this, we must frame our problem in 8854 // terms of figuring out when zero is crossed, instead of when 8855 // Range.getUpper() is crossed. 8856 SmallVector<const SCEV *, 4> NewOps(op_begin(), op_end()); 8857 NewOps[0] = SE.getNegativeSCEV(SE.getConstant(Range.getUpper())); 8858 const SCEV *NewAddRec = SE.getAddRecExpr(NewOps, getLoop(), 8859 // getNoWrapFlags(FlagNW) 8860 FlagAnyWrap); 8861 8862 // Next, solve the constructed addrec 8863 if (auto Roots = 8864 SolveQuadraticEquation(cast<SCEVAddRecExpr>(NewAddRec), SE)) { 8865 const SCEVConstant *R1 = Roots->first; 8866 const SCEVConstant *R2 = Roots->second; 8867 // Pick the smallest positive root value. 8868 if (ConstantInt *CB = dyn_cast<ConstantInt>(ConstantExpr::getICmp( 8869 ICmpInst::ICMP_ULT, R1->getValue(), R2->getValue()))) { 8870 if (!CB->getZExtValue()) 8871 std::swap(R1, R2); // R1 is the minimum root now. 8872 8873 // Make sure the root is not off by one. The returned iteration should 8874 // not be in the range, but the previous one should be. When solving 8875 // for "X*X < 5", for example, we should not return a root of 2. 8876 ConstantInt *R1Val = 8877 EvaluateConstantChrecAtConstant(this, R1->getValue(), SE); 8878 if (Range.contains(R1Val->getValue())) { 8879 // The next iteration must be out of the range... 8880 ConstantInt *NextVal = 8881 ConstantInt::get(SE.getContext(), R1->getAPInt() + 1); 8882 8883 R1Val = EvaluateConstantChrecAtConstant(this, NextVal, SE); 8884 if (!Range.contains(R1Val->getValue())) 8885 return SE.getConstant(NextVal); 8886 return SE.getCouldNotCompute(); // Something strange happened 8887 } 8888 8889 // If R1 was not in the range, then it is a good return value. Make 8890 // sure that R1-1 WAS in the range though, just in case. 8891 ConstantInt *NextVal = 8892 ConstantInt::get(SE.getContext(), R1->getAPInt() - 1); 8893 R1Val = EvaluateConstantChrecAtConstant(this, NextVal, SE); 8894 if (Range.contains(R1Val->getValue())) 8895 return R1; 8896 return SE.getCouldNotCompute(); // Something strange happened 8897 } 8898 } 8899 } 8900 8901 return SE.getCouldNotCompute(); 8902 } 8903 8904 namespace { 8905 struct FindUndefs { 8906 bool Found; 8907 FindUndefs() : Found(false) {} 8908 8909 bool follow(const SCEV *S) { 8910 if (const SCEVUnknown *C = dyn_cast<SCEVUnknown>(S)) { 8911 if (isa<UndefValue>(C->getValue())) 8912 Found = true; 8913 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) { 8914 if (isa<UndefValue>(C->getValue())) 8915 Found = true; 8916 } 8917 8918 // Keep looking if we haven't found it yet. 8919 return !Found; 8920 } 8921 bool isDone() const { 8922 // Stop recursion if we have found an undef. 8923 return Found; 8924 } 8925 }; 8926 } 8927 8928 // Return true when S contains at least an undef value. 8929 static inline bool 8930 containsUndefs(const SCEV *S) { 8931 FindUndefs F; 8932 SCEVTraversal<FindUndefs> ST(F); 8933 ST.visitAll(S); 8934 8935 return F.Found; 8936 } 8937 8938 namespace { 8939 // Collect all steps of SCEV expressions. 8940 struct SCEVCollectStrides { 8941 ScalarEvolution &SE; 8942 SmallVectorImpl<const SCEV *> &Strides; 8943 8944 SCEVCollectStrides(ScalarEvolution &SE, SmallVectorImpl<const SCEV *> &S) 8945 : SE(SE), Strides(S) {} 8946 8947 bool follow(const SCEV *S) { 8948 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) 8949 Strides.push_back(AR->getStepRecurrence(SE)); 8950 return true; 8951 } 8952 bool isDone() const { return false; } 8953 }; 8954 8955 // Collect all SCEVUnknown and SCEVMulExpr expressions. 8956 struct SCEVCollectTerms { 8957 SmallVectorImpl<const SCEV *> &Terms; 8958 8959 SCEVCollectTerms(SmallVectorImpl<const SCEV *> &T) 8960 : Terms(T) {} 8961 8962 bool follow(const SCEV *S) { 8963 if (isa<SCEVUnknown>(S) || isa<SCEVMulExpr>(S)) { 8964 if (!containsUndefs(S)) 8965 Terms.push_back(S); 8966 8967 // Stop recursion: once we collected a term, do not walk its operands. 8968 return false; 8969 } 8970 8971 // Keep looking. 8972 return true; 8973 } 8974 bool isDone() const { return false; } 8975 }; 8976 8977 // Check if a SCEV contains an AddRecExpr. 8978 struct SCEVHasAddRec { 8979 bool &ContainsAddRec; 8980 8981 SCEVHasAddRec(bool &ContainsAddRec) : ContainsAddRec(ContainsAddRec) { 8982 ContainsAddRec = false; 8983 } 8984 8985 bool follow(const SCEV *S) { 8986 if (isa<SCEVAddRecExpr>(S)) { 8987 ContainsAddRec = true; 8988 8989 // Stop recursion: once we collected a term, do not walk its operands. 8990 return false; 8991 } 8992 8993 // Keep looking. 8994 return true; 8995 } 8996 bool isDone() const { return false; } 8997 }; 8998 8999 // Find factors that are multiplied with an expression that (possibly as a 9000 // subexpression) contains an AddRecExpr. In the expression: 9001 // 9002 // 8 * (100 + %p * %q * (%a + {0, +, 1}_loop)) 9003 // 9004 // "%p * %q" are factors multiplied by the expression "(%a + {0, +, 1}_loop)" 9005 // that contains the AddRec {0, +, 1}_loop. %p * %q are likely to be array size 9006 // parameters as they form a product with an induction variable. 9007 // 9008 // This collector expects all array size parameters to be in the same MulExpr. 9009 // It might be necessary to later add support for collecting parameters that are 9010 // spread over different nested MulExpr. 9011 struct SCEVCollectAddRecMultiplies { 9012 SmallVectorImpl<const SCEV *> &Terms; 9013 ScalarEvolution &SE; 9014 9015 SCEVCollectAddRecMultiplies(SmallVectorImpl<const SCEV *> &T, ScalarEvolution &SE) 9016 : Terms(T), SE(SE) {} 9017 9018 bool follow(const SCEV *S) { 9019 if (auto *Mul = dyn_cast<SCEVMulExpr>(S)) { 9020 bool HasAddRec = false; 9021 SmallVector<const SCEV *, 0> Operands; 9022 for (auto Op : Mul->operands()) { 9023 if (isa<SCEVUnknown>(Op)) { 9024 Operands.push_back(Op); 9025 } else { 9026 bool ContainsAddRec; 9027 SCEVHasAddRec ContiansAddRec(ContainsAddRec); 9028 visitAll(Op, ContiansAddRec); 9029 HasAddRec |= ContainsAddRec; 9030 } 9031 } 9032 if (Operands.size() == 0) 9033 return true; 9034 9035 if (!HasAddRec) 9036 return false; 9037 9038 Terms.push_back(SE.getMulExpr(Operands)); 9039 // Stop recursion: once we collected a term, do not walk its operands. 9040 return false; 9041 } 9042 9043 // Keep looking. 9044 return true; 9045 } 9046 bool isDone() const { return false; } 9047 }; 9048 } 9049 9050 /// Find parametric terms in this SCEVAddRecExpr. We first for parameters in 9051 /// two places: 9052 /// 1) The strides of AddRec expressions. 9053 /// 2) Unknowns that are multiplied with AddRec expressions. 9054 void ScalarEvolution::collectParametricTerms(const SCEV *Expr, 9055 SmallVectorImpl<const SCEV *> &Terms) { 9056 SmallVector<const SCEV *, 4> Strides; 9057 SCEVCollectStrides StrideCollector(*this, Strides); 9058 visitAll(Expr, StrideCollector); 9059 9060 DEBUG({ 9061 dbgs() << "Strides:\n"; 9062 for (const SCEV *S : Strides) 9063 dbgs() << *S << "\n"; 9064 }); 9065 9066 for (const SCEV *S : Strides) { 9067 SCEVCollectTerms TermCollector(Terms); 9068 visitAll(S, TermCollector); 9069 } 9070 9071 DEBUG({ 9072 dbgs() << "Terms:\n"; 9073 for (const SCEV *T : Terms) 9074 dbgs() << *T << "\n"; 9075 }); 9076 9077 SCEVCollectAddRecMultiplies MulCollector(Terms, *this); 9078 visitAll(Expr, MulCollector); 9079 } 9080 9081 static bool findArrayDimensionsRec(ScalarEvolution &SE, 9082 SmallVectorImpl<const SCEV *> &Terms, 9083 SmallVectorImpl<const SCEV *> &Sizes) { 9084 int Last = Terms.size() - 1; 9085 const SCEV *Step = Terms[Last]; 9086 9087 // End of recursion. 9088 if (Last == 0) { 9089 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Step)) { 9090 SmallVector<const SCEV *, 2> Qs; 9091 for (const SCEV *Op : M->operands()) 9092 if (!isa<SCEVConstant>(Op)) 9093 Qs.push_back(Op); 9094 9095 Step = SE.getMulExpr(Qs); 9096 } 9097 9098 Sizes.push_back(Step); 9099 return true; 9100 } 9101 9102 for (const SCEV *&Term : Terms) { 9103 // Normalize the terms before the next call to findArrayDimensionsRec. 9104 const SCEV *Q, *R; 9105 SCEVDivision::divide(SE, Term, Step, &Q, &R); 9106 9107 // Bail out when GCD does not evenly divide one of the terms. 9108 if (!R->isZero()) 9109 return false; 9110 9111 Term = Q; 9112 } 9113 9114 // Remove all SCEVConstants. 9115 Terms.erase(std::remove_if(Terms.begin(), Terms.end(), [](const SCEV *E) { 9116 return isa<SCEVConstant>(E); 9117 }), 9118 Terms.end()); 9119 9120 if (Terms.size() > 0) 9121 if (!findArrayDimensionsRec(SE, Terms, Sizes)) 9122 return false; 9123 9124 Sizes.push_back(Step); 9125 return true; 9126 } 9127 9128 // Returns true when S contains at least a SCEVUnknown parameter. 9129 static inline bool 9130 containsParameters(const SCEV *S) { 9131 struct FindParameter { 9132 bool FoundParameter; 9133 FindParameter() : FoundParameter(false) {} 9134 9135 bool follow(const SCEV *S) { 9136 if (isa<SCEVUnknown>(S)) { 9137 FoundParameter = true; 9138 // Stop recursion: we found a parameter. 9139 return false; 9140 } 9141 // Keep looking. 9142 return true; 9143 } 9144 bool isDone() const { 9145 // Stop recursion if we have found a parameter. 9146 return FoundParameter; 9147 } 9148 }; 9149 9150 FindParameter F; 9151 SCEVTraversal<FindParameter> ST(F); 9152 ST.visitAll(S); 9153 9154 return F.FoundParameter; 9155 } 9156 9157 // Returns true when one of the SCEVs of Terms contains a SCEVUnknown parameter. 9158 static inline bool 9159 containsParameters(SmallVectorImpl<const SCEV *> &Terms) { 9160 for (const SCEV *T : Terms) 9161 if (containsParameters(T)) 9162 return true; 9163 return false; 9164 } 9165 9166 // Return the number of product terms in S. 9167 static inline int numberOfTerms(const SCEV *S) { 9168 if (const SCEVMulExpr *Expr = dyn_cast<SCEVMulExpr>(S)) 9169 return Expr->getNumOperands(); 9170 return 1; 9171 } 9172 9173 static const SCEV *removeConstantFactors(ScalarEvolution &SE, const SCEV *T) { 9174 if (isa<SCEVConstant>(T)) 9175 return nullptr; 9176 9177 if (isa<SCEVUnknown>(T)) 9178 return T; 9179 9180 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(T)) { 9181 SmallVector<const SCEV *, 2> Factors; 9182 for (const SCEV *Op : M->operands()) 9183 if (!isa<SCEVConstant>(Op)) 9184 Factors.push_back(Op); 9185 9186 return SE.getMulExpr(Factors); 9187 } 9188 9189 return T; 9190 } 9191 9192 /// Return the size of an element read or written by Inst. 9193 const SCEV *ScalarEvolution::getElementSize(Instruction *Inst) { 9194 Type *Ty; 9195 if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) 9196 Ty = Store->getValueOperand()->getType(); 9197 else if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) 9198 Ty = Load->getType(); 9199 else 9200 return nullptr; 9201 9202 Type *ETy = getEffectiveSCEVType(PointerType::getUnqual(Ty)); 9203 return getSizeOfExpr(ETy, Ty); 9204 } 9205 9206 void ScalarEvolution::findArrayDimensions(SmallVectorImpl<const SCEV *> &Terms, 9207 SmallVectorImpl<const SCEV *> &Sizes, 9208 const SCEV *ElementSize) const { 9209 if (Terms.size() < 1 || !ElementSize) 9210 return; 9211 9212 // Early return when Terms do not contain parameters: we do not delinearize 9213 // non parametric SCEVs. 9214 if (!containsParameters(Terms)) 9215 return; 9216 9217 DEBUG({ 9218 dbgs() << "Terms:\n"; 9219 for (const SCEV *T : Terms) 9220 dbgs() << *T << "\n"; 9221 }); 9222 9223 // Remove duplicates. 9224 std::sort(Terms.begin(), Terms.end()); 9225 Terms.erase(std::unique(Terms.begin(), Terms.end()), Terms.end()); 9226 9227 // Put larger terms first. 9228 std::sort(Terms.begin(), Terms.end(), [](const SCEV *LHS, const SCEV *RHS) { 9229 return numberOfTerms(LHS) > numberOfTerms(RHS); 9230 }); 9231 9232 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this); 9233 9234 // Try to divide all terms by the element size. If term is not divisible by 9235 // element size, proceed with the original term. 9236 for (const SCEV *&Term : Terms) { 9237 const SCEV *Q, *R; 9238 SCEVDivision::divide(SE, Term, ElementSize, &Q, &R); 9239 if (!Q->isZero()) 9240 Term = Q; 9241 } 9242 9243 SmallVector<const SCEV *, 4> NewTerms; 9244 9245 // Remove constant factors. 9246 for (const SCEV *T : Terms) 9247 if (const SCEV *NewT = removeConstantFactors(SE, T)) 9248 NewTerms.push_back(NewT); 9249 9250 DEBUG({ 9251 dbgs() << "Terms after sorting:\n"; 9252 for (const SCEV *T : NewTerms) 9253 dbgs() << *T << "\n"; 9254 }); 9255 9256 if (NewTerms.empty() || 9257 !findArrayDimensionsRec(SE, NewTerms, Sizes)) { 9258 Sizes.clear(); 9259 return; 9260 } 9261 9262 // The last element to be pushed into Sizes is the size of an element. 9263 Sizes.push_back(ElementSize); 9264 9265 DEBUG({ 9266 dbgs() << "Sizes:\n"; 9267 for (const SCEV *S : Sizes) 9268 dbgs() << *S << "\n"; 9269 }); 9270 } 9271 9272 void ScalarEvolution::computeAccessFunctions( 9273 const SCEV *Expr, SmallVectorImpl<const SCEV *> &Subscripts, 9274 SmallVectorImpl<const SCEV *> &Sizes) { 9275 9276 // Early exit in case this SCEV is not an affine multivariate function. 9277 if (Sizes.empty()) 9278 return; 9279 9280 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Expr)) 9281 if (!AR->isAffine()) 9282 return; 9283 9284 const SCEV *Res = Expr; 9285 int Last = Sizes.size() - 1; 9286 for (int i = Last; i >= 0; i--) { 9287 const SCEV *Q, *R; 9288 SCEVDivision::divide(*this, Res, Sizes[i], &Q, &R); 9289 9290 DEBUG({ 9291 dbgs() << "Res: " << *Res << "\n"; 9292 dbgs() << "Sizes[i]: " << *Sizes[i] << "\n"; 9293 dbgs() << "Res divided by Sizes[i]:\n"; 9294 dbgs() << "Quotient: " << *Q << "\n"; 9295 dbgs() << "Remainder: " << *R << "\n"; 9296 }); 9297 9298 Res = Q; 9299 9300 // Do not record the last subscript corresponding to the size of elements in 9301 // the array. 9302 if (i == Last) { 9303 9304 // Bail out if the remainder is too complex. 9305 if (isa<SCEVAddRecExpr>(R)) { 9306 Subscripts.clear(); 9307 Sizes.clear(); 9308 return; 9309 } 9310 9311 continue; 9312 } 9313 9314 // Record the access function for the current subscript. 9315 Subscripts.push_back(R); 9316 } 9317 9318 // Also push in last position the remainder of the last division: it will be 9319 // the access function of the innermost dimension. 9320 Subscripts.push_back(Res); 9321 9322 std::reverse(Subscripts.begin(), Subscripts.end()); 9323 9324 DEBUG({ 9325 dbgs() << "Subscripts:\n"; 9326 for (const SCEV *S : Subscripts) 9327 dbgs() << *S << "\n"; 9328 }); 9329 } 9330 9331 /// Splits the SCEV into two vectors of SCEVs representing the subscripts and 9332 /// sizes of an array access. Returns the remainder of the delinearization that 9333 /// is the offset start of the array. The SCEV->delinearize algorithm computes 9334 /// the multiples of SCEV coefficients: that is a pattern matching of sub 9335 /// expressions in the stride and base of a SCEV corresponding to the 9336 /// computation of a GCD (greatest common divisor) of base and stride. When 9337 /// SCEV->delinearize fails, it returns the SCEV unchanged. 9338 /// 9339 /// For example: when analyzing the memory access A[i][j][k] in this loop nest 9340 /// 9341 /// void foo(long n, long m, long o, double A[n][m][o]) { 9342 /// 9343 /// for (long i = 0; i < n; i++) 9344 /// for (long j = 0; j < m; j++) 9345 /// for (long k = 0; k < o; k++) 9346 /// A[i][j][k] = 1.0; 9347 /// } 9348 /// 9349 /// the delinearization input is the following AddRec SCEV: 9350 /// 9351 /// AddRec: {{{%A,+,(8 * %m * %o)}<%for.i>,+,(8 * %o)}<%for.j>,+,8}<%for.k> 9352 /// 9353 /// From this SCEV, we are able to say that the base offset of the access is %A 9354 /// because it appears as an offset that does not divide any of the strides in 9355 /// the loops: 9356 /// 9357 /// CHECK: Base offset: %A 9358 /// 9359 /// and then SCEV->delinearize determines the size of some of the dimensions of 9360 /// the array as these are the multiples by which the strides are happening: 9361 /// 9362 /// CHECK: ArrayDecl[UnknownSize][%m][%o] with elements of sizeof(double) bytes. 9363 /// 9364 /// Note that the outermost dimension remains of UnknownSize because there are 9365 /// no strides that would help identifying the size of the last dimension: when 9366 /// the array has been statically allocated, one could compute the size of that 9367 /// dimension by dividing the overall size of the array by the size of the known 9368 /// dimensions: %m * %o * 8. 9369 /// 9370 /// Finally delinearize provides the access functions for the array reference 9371 /// that does correspond to A[i][j][k] of the above C testcase: 9372 /// 9373 /// CHECK: ArrayRef[{0,+,1}<%for.i>][{0,+,1}<%for.j>][{0,+,1}<%for.k>] 9374 /// 9375 /// The testcases are checking the output of a function pass: 9376 /// DelinearizationPass that walks through all loads and stores of a function 9377 /// asking for the SCEV of the memory access with respect to all enclosing 9378 /// loops, calling SCEV->delinearize on that and printing the results. 9379 9380 void ScalarEvolution::delinearize(const SCEV *Expr, 9381 SmallVectorImpl<const SCEV *> &Subscripts, 9382 SmallVectorImpl<const SCEV *> &Sizes, 9383 const SCEV *ElementSize) { 9384 // First step: collect parametric terms. 9385 SmallVector<const SCEV *, 4> Terms; 9386 collectParametricTerms(Expr, Terms); 9387 9388 if (Terms.empty()) 9389 return; 9390 9391 // Second step: find subscript sizes. 9392 findArrayDimensions(Terms, Sizes, ElementSize); 9393 9394 if (Sizes.empty()) 9395 return; 9396 9397 // Third step: compute the access functions for each subscript. 9398 computeAccessFunctions(Expr, Subscripts, Sizes); 9399 9400 if (Subscripts.empty()) 9401 return; 9402 9403 DEBUG({ 9404 dbgs() << "succeeded to delinearize " << *Expr << "\n"; 9405 dbgs() << "ArrayDecl[UnknownSize]"; 9406 for (const SCEV *S : Sizes) 9407 dbgs() << "[" << *S << "]"; 9408 9409 dbgs() << "\nArrayRef"; 9410 for (const SCEV *S : Subscripts) 9411 dbgs() << "[" << *S << "]"; 9412 dbgs() << "\n"; 9413 }); 9414 } 9415 9416 //===----------------------------------------------------------------------===// 9417 // SCEVCallbackVH Class Implementation 9418 //===----------------------------------------------------------------------===// 9419 9420 void ScalarEvolution::SCEVCallbackVH::deleted() { 9421 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!"); 9422 if (PHINode *PN = dyn_cast<PHINode>(getValPtr())) 9423 SE->ConstantEvolutionLoopExitValue.erase(PN); 9424 SE->eraseValueFromMap(getValPtr()); 9425 // this now dangles! 9426 } 9427 9428 void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(Value *V) { 9429 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!"); 9430 9431 // Forget all the expressions associated with users of the old value, 9432 // so that future queries will recompute the expressions using the new 9433 // value. 9434 Value *Old = getValPtr(); 9435 SmallVector<User *, 16> Worklist(Old->user_begin(), Old->user_end()); 9436 SmallPtrSet<User *, 8> Visited; 9437 while (!Worklist.empty()) { 9438 User *U = Worklist.pop_back_val(); 9439 // Deleting the Old value will cause this to dangle. Postpone 9440 // that until everything else is done. 9441 if (U == Old) 9442 continue; 9443 if (!Visited.insert(U).second) 9444 continue; 9445 if (PHINode *PN = dyn_cast<PHINode>(U)) 9446 SE->ConstantEvolutionLoopExitValue.erase(PN); 9447 SE->eraseValueFromMap(U); 9448 Worklist.insert(Worklist.end(), U->user_begin(), U->user_end()); 9449 } 9450 // Delete the Old value. 9451 if (PHINode *PN = dyn_cast<PHINode>(Old)) 9452 SE->ConstantEvolutionLoopExitValue.erase(PN); 9453 SE->eraseValueFromMap(Old); 9454 // this now dangles! 9455 } 9456 9457 ScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se) 9458 : CallbackVH(V), SE(se) {} 9459 9460 //===----------------------------------------------------------------------===// 9461 // ScalarEvolution Class Implementation 9462 //===----------------------------------------------------------------------===// 9463 9464 ScalarEvolution::ScalarEvolution(Function &F, TargetLibraryInfo &TLI, 9465 AssumptionCache &AC, DominatorTree &DT, 9466 LoopInfo &LI) 9467 : F(F), TLI(TLI), AC(AC), DT(DT), LI(LI), 9468 CouldNotCompute(new SCEVCouldNotCompute()), 9469 WalkingBEDominatingConds(false), ProvingSplitPredicate(false), 9470 ValuesAtScopes(64), LoopDispositions(64), BlockDispositions(64), 9471 FirstUnknown(nullptr) { 9472 9473 // To use guards for proving predicates, we need to scan every instruction in 9474 // relevant basic blocks, and not just terminators. Doing this is a waste of 9475 // time if the IR does not actually contain any calls to 9476 // @llvm.experimental.guard, so do a quick check and remember this beforehand. 9477 // 9478 // This pessimizes the case where a pass that preserves ScalarEvolution wants 9479 // to _add_ guards to the module when there weren't any before, and wants 9480 // ScalarEvolution to optimize based on those guards. For now we prefer to be 9481 // efficient in lieu of being smart in that rather obscure case. 9482 9483 auto *GuardDecl = F.getParent()->getFunction( 9484 Intrinsic::getName(Intrinsic::experimental_guard)); 9485 HasGuards = GuardDecl && !GuardDecl->use_empty(); 9486 } 9487 9488 ScalarEvolution::ScalarEvolution(ScalarEvolution &&Arg) 9489 : F(Arg.F), HasGuards(Arg.HasGuards), TLI(Arg.TLI), AC(Arg.AC), DT(Arg.DT), 9490 LI(Arg.LI), CouldNotCompute(std::move(Arg.CouldNotCompute)), 9491 ValueExprMap(std::move(Arg.ValueExprMap)), 9492 WalkingBEDominatingConds(false), ProvingSplitPredicate(false), 9493 BackedgeTakenCounts(std::move(Arg.BackedgeTakenCounts)), 9494 PredicatedBackedgeTakenCounts( 9495 std::move(Arg.PredicatedBackedgeTakenCounts)), 9496 ConstantEvolutionLoopExitValue( 9497 std::move(Arg.ConstantEvolutionLoopExitValue)), 9498 ValuesAtScopes(std::move(Arg.ValuesAtScopes)), 9499 LoopDispositions(std::move(Arg.LoopDispositions)), 9500 BlockDispositions(std::move(Arg.BlockDispositions)), 9501 UnsignedRanges(std::move(Arg.UnsignedRanges)), 9502 SignedRanges(std::move(Arg.SignedRanges)), 9503 UniqueSCEVs(std::move(Arg.UniqueSCEVs)), 9504 UniquePreds(std::move(Arg.UniquePreds)), 9505 SCEVAllocator(std::move(Arg.SCEVAllocator)), 9506 FirstUnknown(Arg.FirstUnknown) { 9507 Arg.FirstUnknown = nullptr; 9508 } 9509 9510 ScalarEvolution::~ScalarEvolution() { 9511 // Iterate through all the SCEVUnknown instances and call their 9512 // destructors, so that they release their references to their values. 9513 for (SCEVUnknown *U = FirstUnknown; U;) { 9514 SCEVUnknown *Tmp = U; 9515 U = U->Next; 9516 Tmp->~SCEVUnknown(); 9517 } 9518 FirstUnknown = nullptr; 9519 9520 ExprValueMap.clear(); 9521 ValueExprMap.clear(); 9522 HasRecMap.clear(); 9523 9524 // Free any extra memory created for ExitNotTakenInfo in the unlikely event 9525 // that a loop had multiple computable exits. 9526 for (auto &BTCI : BackedgeTakenCounts) 9527 BTCI.second.clear(); 9528 for (auto &BTCI : PredicatedBackedgeTakenCounts) 9529 BTCI.second.clear(); 9530 9531 assert(PendingLoopPredicates.empty() && "isImpliedCond garbage"); 9532 assert(!WalkingBEDominatingConds && "isLoopBackedgeGuardedByCond garbage!"); 9533 assert(!ProvingSplitPredicate && "ProvingSplitPredicate garbage!"); 9534 } 9535 9536 bool ScalarEvolution::hasLoopInvariantBackedgeTakenCount(const Loop *L) { 9537 return !isa<SCEVCouldNotCompute>(getBackedgeTakenCount(L)); 9538 } 9539 9540 static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, 9541 const Loop *L) { 9542 // Print all inner loops first 9543 for (Loop *I : *L) 9544 PrintLoopInfo(OS, SE, I); 9545 9546 OS << "Loop "; 9547 L->getHeader()->printAsOperand(OS, /*PrintType=*/false); 9548 OS << ": "; 9549 9550 SmallVector<BasicBlock *, 8> ExitBlocks; 9551 L->getExitBlocks(ExitBlocks); 9552 if (ExitBlocks.size() != 1) 9553 OS << "<multiple exits> "; 9554 9555 if (SE->hasLoopInvariantBackedgeTakenCount(L)) { 9556 OS << "backedge-taken count is " << *SE->getBackedgeTakenCount(L); 9557 } else { 9558 OS << "Unpredictable backedge-taken count. "; 9559 } 9560 9561 OS << "\n" 9562 "Loop "; 9563 L->getHeader()->printAsOperand(OS, /*PrintType=*/false); 9564 OS << ": "; 9565 9566 if (!isa<SCEVCouldNotCompute>(SE->getMaxBackedgeTakenCount(L))) { 9567 OS << "max backedge-taken count is " << *SE->getMaxBackedgeTakenCount(L); 9568 } else { 9569 OS << "Unpredictable max backedge-taken count. "; 9570 } 9571 9572 OS << "\n" 9573 "Loop "; 9574 L->getHeader()->printAsOperand(OS, /*PrintType=*/false); 9575 OS << ": "; 9576 9577 SCEVUnionPredicate Pred; 9578 auto PBT = SE->getPredicatedBackedgeTakenCount(L, Pred); 9579 if (!isa<SCEVCouldNotCompute>(PBT)) { 9580 OS << "Predicated backedge-taken count is " << *PBT << "\n"; 9581 OS << " Predicates:\n"; 9582 Pred.print(OS, 4); 9583 } else { 9584 OS << "Unpredictable predicated backedge-taken count. "; 9585 } 9586 OS << "\n"; 9587 } 9588 9589 static StringRef loopDispositionToStr(ScalarEvolution::LoopDisposition LD) { 9590 switch (LD) { 9591 case ScalarEvolution::LoopVariant: 9592 return "Variant"; 9593 case ScalarEvolution::LoopInvariant: 9594 return "Invariant"; 9595 case ScalarEvolution::LoopComputable: 9596 return "Computable"; 9597 } 9598 llvm_unreachable("Unknown ScalarEvolution::LoopDisposition kind!"); 9599 } 9600 9601 void ScalarEvolution::print(raw_ostream &OS) const { 9602 // ScalarEvolution's implementation of the print method is to print 9603 // out SCEV values of all instructions that are interesting. Doing 9604 // this potentially causes it to create new SCEV objects though, 9605 // which technically conflicts with the const qualifier. This isn't 9606 // observable from outside the class though, so casting away the 9607 // const isn't dangerous. 9608 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this); 9609 9610 OS << "Classifying expressions for: "; 9611 F.printAsOperand(OS, /*PrintType=*/false); 9612 OS << "\n"; 9613 for (Instruction &I : instructions(F)) 9614 if (isSCEVable(I.getType()) && !isa<CmpInst>(I)) { 9615 OS << I << '\n'; 9616 OS << " --> "; 9617 const SCEV *SV = SE.getSCEV(&I); 9618 SV->print(OS); 9619 if (!isa<SCEVCouldNotCompute>(SV)) { 9620 OS << " U: "; 9621 SE.getUnsignedRange(SV).print(OS); 9622 OS << " S: "; 9623 SE.getSignedRange(SV).print(OS); 9624 } 9625 9626 const Loop *L = LI.getLoopFor(I.getParent()); 9627 9628 const SCEV *AtUse = SE.getSCEVAtScope(SV, L); 9629 if (AtUse != SV) { 9630 OS << " --> "; 9631 AtUse->print(OS); 9632 if (!isa<SCEVCouldNotCompute>(AtUse)) { 9633 OS << " U: "; 9634 SE.getUnsignedRange(AtUse).print(OS); 9635 OS << " S: "; 9636 SE.getSignedRange(AtUse).print(OS); 9637 } 9638 } 9639 9640 if (L) { 9641 OS << "\t\t" "Exits: "; 9642 const SCEV *ExitValue = SE.getSCEVAtScope(SV, L->getParentLoop()); 9643 if (!SE.isLoopInvariant(ExitValue, L)) { 9644 OS << "<<Unknown>>"; 9645 } else { 9646 OS << *ExitValue; 9647 } 9648 9649 bool First = true; 9650 for (auto *Iter = L; Iter; Iter = Iter->getParentLoop()) { 9651 if (First) { 9652 OS << "\t\t" "LoopDispositions: { "; 9653 First = false; 9654 } else { 9655 OS << ", "; 9656 } 9657 9658 Iter->getHeader()->printAsOperand(OS, /*PrintType=*/false); 9659 OS << ": " << loopDispositionToStr(SE.getLoopDisposition(SV, Iter)); 9660 } 9661 9662 for (auto *InnerL : depth_first(L)) { 9663 if (InnerL == L) 9664 continue; 9665 if (First) { 9666 OS << "\t\t" "LoopDispositions: { "; 9667 First = false; 9668 } else { 9669 OS << ", "; 9670 } 9671 9672 InnerL->getHeader()->printAsOperand(OS, /*PrintType=*/false); 9673 OS << ": " << loopDispositionToStr(SE.getLoopDisposition(SV, InnerL)); 9674 } 9675 9676 OS << " }"; 9677 } 9678 9679 OS << "\n"; 9680 } 9681 9682 OS << "Determining loop execution counts for: "; 9683 F.printAsOperand(OS, /*PrintType=*/false); 9684 OS << "\n"; 9685 for (Loop *I : LI) 9686 PrintLoopInfo(OS, &SE, I); 9687 } 9688 9689 ScalarEvolution::LoopDisposition 9690 ScalarEvolution::getLoopDisposition(const SCEV *S, const Loop *L) { 9691 auto &Values = LoopDispositions[S]; 9692 for (auto &V : Values) { 9693 if (V.getPointer() == L) 9694 return V.getInt(); 9695 } 9696 Values.emplace_back(L, LoopVariant); 9697 LoopDisposition D = computeLoopDisposition(S, L); 9698 auto &Values2 = LoopDispositions[S]; 9699 for (auto &V : make_range(Values2.rbegin(), Values2.rend())) { 9700 if (V.getPointer() == L) { 9701 V.setInt(D); 9702 break; 9703 } 9704 } 9705 return D; 9706 } 9707 9708 ScalarEvolution::LoopDisposition 9709 ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) { 9710 switch (static_cast<SCEVTypes>(S->getSCEVType())) { 9711 case scConstant: 9712 return LoopInvariant; 9713 case scTruncate: 9714 case scZeroExtend: 9715 case scSignExtend: 9716 return getLoopDisposition(cast<SCEVCastExpr>(S)->getOperand(), L); 9717 case scAddRecExpr: { 9718 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S); 9719 9720 // If L is the addrec's loop, it's computable. 9721 if (AR->getLoop() == L) 9722 return LoopComputable; 9723 9724 // Add recurrences are never invariant in the function-body (null loop). 9725 if (!L) 9726 return LoopVariant; 9727 9728 // This recurrence is variant w.r.t. L if L contains AR's loop. 9729 if (L->contains(AR->getLoop())) 9730 return LoopVariant; 9731 9732 // This recurrence is invariant w.r.t. L if AR's loop contains L. 9733 if (AR->getLoop()->contains(L)) 9734 return LoopInvariant; 9735 9736 // This recurrence is variant w.r.t. L if any of its operands 9737 // are variant. 9738 for (auto *Op : AR->operands()) 9739 if (!isLoopInvariant(Op, L)) 9740 return LoopVariant; 9741 9742 // Otherwise it's loop-invariant. 9743 return LoopInvariant; 9744 } 9745 case scAddExpr: 9746 case scMulExpr: 9747 case scUMaxExpr: 9748 case scSMaxExpr: { 9749 bool HasVarying = false; 9750 for (auto *Op : cast<SCEVNAryExpr>(S)->operands()) { 9751 LoopDisposition D = getLoopDisposition(Op, L); 9752 if (D == LoopVariant) 9753 return LoopVariant; 9754 if (D == LoopComputable) 9755 HasVarying = true; 9756 } 9757 return HasVarying ? LoopComputable : LoopInvariant; 9758 } 9759 case scUDivExpr: { 9760 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S); 9761 LoopDisposition LD = getLoopDisposition(UDiv->getLHS(), L); 9762 if (LD == LoopVariant) 9763 return LoopVariant; 9764 LoopDisposition RD = getLoopDisposition(UDiv->getRHS(), L); 9765 if (RD == LoopVariant) 9766 return LoopVariant; 9767 return (LD == LoopInvariant && RD == LoopInvariant) ? 9768 LoopInvariant : LoopComputable; 9769 } 9770 case scUnknown: 9771 // All non-instruction values are loop invariant. All instructions are loop 9772 // invariant if they are not contained in the specified loop. 9773 // Instructions are never considered invariant in the function body 9774 // (null loop) because they are defined within the "loop". 9775 if (auto *I = dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue())) 9776 return (L && !L->contains(I)) ? LoopInvariant : LoopVariant; 9777 return LoopInvariant; 9778 case scCouldNotCompute: 9779 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 9780 } 9781 llvm_unreachable("Unknown SCEV kind!"); 9782 } 9783 9784 bool ScalarEvolution::isLoopInvariant(const SCEV *S, const Loop *L) { 9785 return getLoopDisposition(S, L) == LoopInvariant; 9786 } 9787 9788 bool ScalarEvolution::hasComputableLoopEvolution(const SCEV *S, const Loop *L) { 9789 return getLoopDisposition(S, L) == LoopComputable; 9790 } 9791 9792 ScalarEvolution::BlockDisposition 9793 ScalarEvolution::getBlockDisposition(const SCEV *S, const BasicBlock *BB) { 9794 auto &Values = BlockDispositions[S]; 9795 for (auto &V : Values) { 9796 if (V.getPointer() == BB) 9797 return V.getInt(); 9798 } 9799 Values.emplace_back(BB, DoesNotDominateBlock); 9800 BlockDisposition D = computeBlockDisposition(S, BB); 9801 auto &Values2 = BlockDispositions[S]; 9802 for (auto &V : make_range(Values2.rbegin(), Values2.rend())) { 9803 if (V.getPointer() == BB) { 9804 V.setInt(D); 9805 break; 9806 } 9807 } 9808 return D; 9809 } 9810 9811 ScalarEvolution::BlockDisposition 9812 ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) { 9813 switch (static_cast<SCEVTypes>(S->getSCEVType())) { 9814 case scConstant: 9815 return ProperlyDominatesBlock; 9816 case scTruncate: 9817 case scZeroExtend: 9818 case scSignExtend: 9819 return getBlockDisposition(cast<SCEVCastExpr>(S)->getOperand(), BB); 9820 case scAddRecExpr: { 9821 // This uses a "dominates" query instead of "properly dominates" query 9822 // to test for proper dominance too, because the instruction which 9823 // produces the addrec's value is a PHI, and a PHI effectively properly 9824 // dominates its entire containing block. 9825 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S); 9826 if (!DT.dominates(AR->getLoop()->getHeader(), BB)) 9827 return DoesNotDominateBlock; 9828 } 9829 // FALL THROUGH into SCEVNAryExpr handling. 9830 case scAddExpr: 9831 case scMulExpr: 9832 case scUMaxExpr: 9833 case scSMaxExpr: { 9834 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(S); 9835 bool Proper = true; 9836 for (const SCEV *NAryOp : NAry->operands()) { 9837 BlockDisposition D = getBlockDisposition(NAryOp, BB); 9838 if (D == DoesNotDominateBlock) 9839 return DoesNotDominateBlock; 9840 if (D == DominatesBlock) 9841 Proper = false; 9842 } 9843 return Proper ? ProperlyDominatesBlock : DominatesBlock; 9844 } 9845 case scUDivExpr: { 9846 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S); 9847 const SCEV *LHS = UDiv->getLHS(), *RHS = UDiv->getRHS(); 9848 BlockDisposition LD = getBlockDisposition(LHS, BB); 9849 if (LD == DoesNotDominateBlock) 9850 return DoesNotDominateBlock; 9851 BlockDisposition RD = getBlockDisposition(RHS, BB); 9852 if (RD == DoesNotDominateBlock) 9853 return DoesNotDominateBlock; 9854 return (LD == ProperlyDominatesBlock && RD == ProperlyDominatesBlock) ? 9855 ProperlyDominatesBlock : DominatesBlock; 9856 } 9857 case scUnknown: 9858 if (Instruction *I = 9859 dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue())) { 9860 if (I->getParent() == BB) 9861 return DominatesBlock; 9862 if (DT.properlyDominates(I->getParent(), BB)) 9863 return ProperlyDominatesBlock; 9864 return DoesNotDominateBlock; 9865 } 9866 return ProperlyDominatesBlock; 9867 case scCouldNotCompute: 9868 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 9869 } 9870 llvm_unreachable("Unknown SCEV kind!"); 9871 } 9872 9873 bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) { 9874 return getBlockDisposition(S, BB) >= DominatesBlock; 9875 } 9876 9877 bool ScalarEvolution::properlyDominates(const SCEV *S, const BasicBlock *BB) { 9878 return getBlockDisposition(S, BB) == ProperlyDominatesBlock; 9879 } 9880 9881 bool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const { 9882 // Search for a SCEV expression node within an expression tree. 9883 // Implements SCEVTraversal::Visitor. 9884 struct SCEVSearch { 9885 const SCEV *Node; 9886 bool IsFound; 9887 9888 SCEVSearch(const SCEV *N): Node(N), IsFound(false) {} 9889 9890 bool follow(const SCEV *S) { 9891 IsFound |= (S == Node); 9892 return !IsFound; 9893 } 9894 bool isDone() const { return IsFound; } 9895 }; 9896 9897 SCEVSearch Search(Op); 9898 visitAll(S, Search); 9899 return Search.IsFound; 9900 } 9901 9902 void ScalarEvolution::forgetMemoizedResults(const SCEV *S) { 9903 ValuesAtScopes.erase(S); 9904 LoopDispositions.erase(S); 9905 BlockDispositions.erase(S); 9906 UnsignedRanges.erase(S); 9907 SignedRanges.erase(S); 9908 ExprValueMap.erase(S); 9909 HasRecMap.erase(S); 9910 9911 auto RemoveSCEVFromBackedgeMap = 9912 [S, this](DenseMap<const Loop *, BackedgeTakenInfo> &Map) { 9913 for (auto I = Map.begin(), E = Map.end(); I != E;) { 9914 BackedgeTakenInfo &BEInfo = I->second; 9915 if (BEInfo.hasOperand(S, this)) { 9916 BEInfo.clear(); 9917 Map.erase(I++); 9918 } else 9919 ++I; 9920 } 9921 }; 9922 9923 RemoveSCEVFromBackedgeMap(BackedgeTakenCounts); 9924 RemoveSCEVFromBackedgeMap(PredicatedBackedgeTakenCounts); 9925 } 9926 9927 typedef DenseMap<const Loop *, std::string> VerifyMap; 9928 9929 /// replaceSubString - Replaces all occurrences of From in Str with To. 9930 static void replaceSubString(std::string &Str, StringRef From, StringRef To) { 9931 size_t Pos = 0; 9932 while ((Pos = Str.find(From, Pos)) != std::string::npos) { 9933 Str.replace(Pos, From.size(), To.data(), To.size()); 9934 Pos += To.size(); 9935 } 9936 } 9937 9938 /// getLoopBackedgeTakenCounts - Helper method for verifyAnalysis. 9939 static void 9940 getLoopBackedgeTakenCounts(Loop *L, VerifyMap &Map, ScalarEvolution &SE) { 9941 std::string &S = Map[L]; 9942 if (S.empty()) { 9943 raw_string_ostream OS(S); 9944 SE.getBackedgeTakenCount(L)->print(OS); 9945 9946 // false and 0 are semantically equivalent. This can happen in dead loops. 9947 replaceSubString(OS.str(), "false", "0"); 9948 // Remove wrap flags, their use in SCEV is highly fragile. 9949 // FIXME: Remove this when SCEV gets smarter about them. 9950 replaceSubString(OS.str(), "<nw>", ""); 9951 replaceSubString(OS.str(), "<nsw>", ""); 9952 replaceSubString(OS.str(), "<nuw>", ""); 9953 } 9954 9955 for (auto *R : reverse(*L)) 9956 getLoopBackedgeTakenCounts(R, Map, SE); // recurse. 9957 } 9958 9959 void ScalarEvolution::verify() const { 9960 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this); 9961 9962 // Gather stringified backedge taken counts for all loops using SCEV's caches. 9963 // FIXME: It would be much better to store actual values instead of strings, 9964 // but SCEV pointers will change if we drop the caches. 9965 VerifyMap BackedgeDumpsOld, BackedgeDumpsNew; 9966 for (LoopInfo::reverse_iterator I = LI.rbegin(), E = LI.rend(); I != E; ++I) 9967 getLoopBackedgeTakenCounts(*I, BackedgeDumpsOld, SE); 9968 9969 // Gather stringified backedge taken counts for all loops using a fresh 9970 // ScalarEvolution object. 9971 ScalarEvolution SE2(F, TLI, AC, DT, LI); 9972 for (LoopInfo::reverse_iterator I = LI.rbegin(), E = LI.rend(); I != E; ++I) 9973 getLoopBackedgeTakenCounts(*I, BackedgeDumpsNew, SE2); 9974 9975 // Now compare whether they're the same with and without caches. This allows 9976 // verifying that no pass changed the cache. 9977 assert(BackedgeDumpsOld.size() == BackedgeDumpsNew.size() && 9978 "New loops suddenly appeared!"); 9979 9980 for (VerifyMap::iterator OldI = BackedgeDumpsOld.begin(), 9981 OldE = BackedgeDumpsOld.end(), 9982 NewI = BackedgeDumpsNew.begin(); 9983 OldI != OldE; ++OldI, ++NewI) { 9984 assert(OldI->first == NewI->first && "Loop order changed!"); 9985 9986 // Compare the stringified SCEVs. We don't care if undef backedgetaken count 9987 // changes. 9988 // FIXME: We currently ignore SCEV changes from/to CouldNotCompute. This 9989 // means that a pass is buggy or SCEV has to learn a new pattern but is 9990 // usually not harmful. 9991 if (OldI->second != NewI->second && 9992 OldI->second.find("undef") == std::string::npos && 9993 NewI->second.find("undef") == std::string::npos && 9994 OldI->second != "***COULDNOTCOMPUTE***" && 9995 NewI->second != "***COULDNOTCOMPUTE***") { 9996 dbgs() << "SCEVValidator: SCEV for loop '" 9997 << OldI->first->getHeader()->getName() 9998 << "' changed from '" << OldI->second 9999 << "' to '" << NewI->second << "'!\n"; 10000 std::abort(); 10001 } 10002 } 10003 10004 // TODO: Verify more things. 10005 } 10006 10007 char ScalarEvolutionAnalysis::PassID; 10008 10009 ScalarEvolution ScalarEvolutionAnalysis::run(Function &F, 10010 AnalysisManager<Function> &AM) { 10011 return ScalarEvolution(F, AM.getResult<TargetLibraryAnalysis>(F), 10012 AM.getResult<AssumptionAnalysis>(F), 10013 AM.getResult<DominatorTreeAnalysis>(F), 10014 AM.getResult<LoopAnalysis>(F)); 10015 } 10016 10017 PreservedAnalyses 10018 ScalarEvolutionPrinterPass::run(Function &F, AnalysisManager<Function> &AM) { 10019 AM.getResult<ScalarEvolutionAnalysis>(F).print(OS); 10020 return PreservedAnalyses::all(); 10021 } 10022 10023 INITIALIZE_PASS_BEGIN(ScalarEvolutionWrapperPass, "scalar-evolution", 10024 "Scalar Evolution Analysis", false, true) 10025 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) 10026 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) 10027 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) 10028 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 10029 INITIALIZE_PASS_END(ScalarEvolutionWrapperPass, "scalar-evolution", 10030 "Scalar Evolution Analysis", false, true) 10031 char ScalarEvolutionWrapperPass::ID = 0; 10032 10033 ScalarEvolutionWrapperPass::ScalarEvolutionWrapperPass() : FunctionPass(ID) { 10034 initializeScalarEvolutionWrapperPassPass(*PassRegistry::getPassRegistry()); 10035 } 10036 10037 bool ScalarEvolutionWrapperPass::runOnFunction(Function &F) { 10038 SE.reset(new ScalarEvolution( 10039 F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(), 10040 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F), 10041 getAnalysis<DominatorTreeWrapperPass>().getDomTree(), 10042 getAnalysis<LoopInfoWrapperPass>().getLoopInfo())); 10043 return false; 10044 } 10045 10046 void ScalarEvolutionWrapperPass::releaseMemory() { SE.reset(); } 10047 10048 void ScalarEvolutionWrapperPass::print(raw_ostream &OS, const Module *) const { 10049 SE->print(OS); 10050 } 10051 10052 void ScalarEvolutionWrapperPass::verifyAnalysis() const { 10053 if (!VerifySCEV) 10054 return; 10055 10056 SE->verify(); 10057 } 10058 10059 void ScalarEvolutionWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const { 10060 AU.setPreservesAll(); 10061 AU.addRequiredTransitive<AssumptionCacheTracker>(); 10062 AU.addRequiredTransitive<LoopInfoWrapperPass>(); 10063 AU.addRequiredTransitive<DominatorTreeWrapperPass>(); 10064 AU.addRequiredTransitive<TargetLibraryInfoWrapperPass>(); 10065 } 10066 10067 const SCEVPredicate * 10068 ScalarEvolution::getEqualPredicate(const SCEVUnknown *LHS, 10069 const SCEVConstant *RHS) { 10070 FoldingSetNodeID ID; 10071 // Unique this node based on the arguments 10072 ID.AddInteger(SCEVPredicate::P_Equal); 10073 ID.AddPointer(LHS); 10074 ID.AddPointer(RHS); 10075 void *IP = nullptr; 10076 if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP)) 10077 return S; 10078 SCEVEqualPredicate *Eq = new (SCEVAllocator) 10079 SCEVEqualPredicate(ID.Intern(SCEVAllocator), LHS, RHS); 10080 UniquePreds.InsertNode(Eq, IP); 10081 return Eq; 10082 } 10083 10084 const SCEVPredicate *ScalarEvolution::getWrapPredicate( 10085 const SCEVAddRecExpr *AR, 10086 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) { 10087 FoldingSetNodeID ID; 10088 // Unique this node based on the arguments 10089 ID.AddInteger(SCEVPredicate::P_Wrap); 10090 ID.AddPointer(AR); 10091 ID.AddInteger(AddedFlags); 10092 void *IP = nullptr; 10093 if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP)) 10094 return S; 10095 auto *OF = new (SCEVAllocator) 10096 SCEVWrapPredicate(ID.Intern(SCEVAllocator), AR, AddedFlags); 10097 UniquePreds.InsertNode(OF, IP); 10098 return OF; 10099 } 10100 10101 namespace { 10102 10103 class SCEVPredicateRewriter : public SCEVRewriteVisitor<SCEVPredicateRewriter> { 10104 public: 10105 // Rewrites \p S in the context of a loop L and the predicate A. 10106 // If Assume is true, rewrite is free to add further predicates to A 10107 // such that the result will be an AddRecExpr. 10108 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE, 10109 SCEVUnionPredicate &A, bool Assume) { 10110 SCEVPredicateRewriter Rewriter(L, SE, A, Assume); 10111 return Rewriter.visit(S); 10112 } 10113 10114 SCEVPredicateRewriter(const Loop *L, ScalarEvolution &SE, 10115 SCEVUnionPredicate &P, bool Assume) 10116 : SCEVRewriteVisitor(SE), P(P), L(L), Assume(Assume) {} 10117 10118 const SCEV *visitUnknown(const SCEVUnknown *Expr) { 10119 auto ExprPreds = P.getPredicatesForExpr(Expr); 10120 for (auto *Pred : ExprPreds) 10121 if (const auto *IPred = dyn_cast<SCEVEqualPredicate>(Pred)) 10122 if (IPred->getLHS() == Expr) 10123 return IPred->getRHS(); 10124 10125 return Expr; 10126 } 10127 10128 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) { 10129 const SCEV *Operand = visit(Expr->getOperand()); 10130 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Operand); 10131 if (AR && AR->getLoop() == L && AR->isAffine()) { 10132 // This couldn't be folded because the operand didn't have the nuw 10133 // flag. Add the nusw flag as an assumption that we could make. 10134 const SCEV *Step = AR->getStepRecurrence(SE); 10135 Type *Ty = Expr->getType(); 10136 if (addOverflowAssumption(AR, SCEVWrapPredicate::IncrementNUSW)) 10137 return SE.getAddRecExpr(SE.getZeroExtendExpr(AR->getStart(), Ty), 10138 SE.getSignExtendExpr(Step, Ty), L, 10139 AR->getNoWrapFlags()); 10140 } 10141 return SE.getZeroExtendExpr(Operand, Expr->getType()); 10142 } 10143 10144 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) { 10145 const SCEV *Operand = visit(Expr->getOperand()); 10146 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Operand); 10147 if (AR && AR->getLoop() == L && AR->isAffine()) { 10148 // This couldn't be folded because the operand didn't have the nsw 10149 // flag. Add the nssw flag as an assumption that we could make. 10150 const SCEV *Step = AR->getStepRecurrence(SE); 10151 Type *Ty = Expr->getType(); 10152 if (addOverflowAssumption(AR, SCEVWrapPredicate::IncrementNSSW)) 10153 return SE.getAddRecExpr(SE.getSignExtendExpr(AR->getStart(), Ty), 10154 SE.getSignExtendExpr(Step, Ty), L, 10155 AR->getNoWrapFlags()); 10156 } 10157 return SE.getSignExtendExpr(Operand, Expr->getType()); 10158 } 10159 10160 private: 10161 bool addOverflowAssumption(const SCEVAddRecExpr *AR, 10162 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) { 10163 auto *A = SE.getWrapPredicate(AR, AddedFlags); 10164 if (!Assume) { 10165 // Check if we've already made this assumption. 10166 if (P.implies(A)) 10167 return true; 10168 return false; 10169 } 10170 P.add(A); 10171 return true; 10172 } 10173 10174 SCEVUnionPredicate &P; 10175 const Loop *L; 10176 bool Assume; 10177 }; 10178 } // end anonymous namespace 10179 10180 const SCEV *ScalarEvolution::rewriteUsingPredicate(const SCEV *S, const Loop *L, 10181 SCEVUnionPredicate &Preds) { 10182 return SCEVPredicateRewriter::rewrite(S, L, *this, Preds, false); 10183 } 10184 10185 const SCEVAddRecExpr * 10186 ScalarEvolution::convertSCEVToAddRecWithPredicates(const SCEV *S, const Loop *L, 10187 SCEVUnionPredicate &Preds) { 10188 SCEVUnionPredicate TransformPreds; 10189 S = SCEVPredicateRewriter::rewrite(S, L, *this, TransformPreds, true); 10190 auto *AddRec = dyn_cast<SCEVAddRecExpr>(S); 10191 10192 if (!AddRec) 10193 return nullptr; 10194 10195 // Since the transformation was successful, we can now transfer the SCEV 10196 // predicates. 10197 Preds.add(&TransformPreds); 10198 return AddRec; 10199 } 10200 10201 /// SCEV predicates 10202 SCEVPredicate::SCEVPredicate(const FoldingSetNodeIDRef ID, 10203 SCEVPredicateKind Kind) 10204 : FastID(ID), Kind(Kind) {} 10205 10206 SCEVEqualPredicate::SCEVEqualPredicate(const FoldingSetNodeIDRef ID, 10207 const SCEVUnknown *LHS, 10208 const SCEVConstant *RHS) 10209 : SCEVPredicate(ID, P_Equal), LHS(LHS), RHS(RHS) {} 10210 10211 bool SCEVEqualPredicate::implies(const SCEVPredicate *N) const { 10212 const auto *Op = dyn_cast<SCEVEqualPredicate>(N); 10213 10214 if (!Op) 10215 return false; 10216 10217 return Op->LHS == LHS && Op->RHS == RHS; 10218 } 10219 10220 bool SCEVEqualPredicate::isAlwaysTrue() const { return false; } 10221 10222 const SCEV *SCEVEqualPredicate::getExpr() const { return LHS; } 10223 10224 void SCEVEqualPredicate::print(raw_ostream &OS, unsigned Depth) const { 10225 OS.indent(Depth) << "Equal predicate: " << *LHS << " == " << *RHS << "\n"; 10226 } 10227 10228 SCEVWrapPredicate::SCEVWrapPredicate(const FoldingSetNodeIDRef ID, 10229 const SCEVAddRecExpr *AR, 10230 IncrementWrapFlags Flags) 10231 : SCEVPredicate(ID, P_Wrap), AR(AR), Flags(Flags) {} 10232 10233 const SCEV *SCEVWrapPredicate::getExpr() const { return AR; } 10234 10235 bool SCEVWrapPredicate::implies(const SCEVPredicate *N) const { 10236 const auto *Op = dyn_cast<SCEVWrapPredicate>(N); 10237 10238 return Op && Op->AR == AR && setFlags(Flags, Op->Flags) == Flags; 10239 } 10240 10241 bool SCEVWrapPredicate::isAlwaysTrue() const { 10242 SCEV::NoWrapFlags ScevFlags = AR->getNoWrapFlags(); 10243 IncrementWrapFlags IFlags = Flags; 10244 10245 if (ScalarEvolution::setFlags(ScevFlags, SCEV::FlagNSW) == ScevFlags) 10246 IFlags = clearFlags(IFlags, IncrementNSSW); 10247 10248 return IFlags == IncrementAnyWrap; 10249 } 10250 10251 void SCEVWrapPredicate::print(raw_ostream &OS, unsigned Depth) const { 10252 OS.indent(Depth) << *getExpr() << " Added Flags: "; 10253 if (SCEVWrapPredicate::IncrementNUSW & getFlags()) 10254 OS << "<nusw>"; 10255 if (SCEVWrapPredicate::IncrementNSSW & getFlags()) 10256 OS << "<nssw>"; 10257 OS << "\n"; 10258 } 10259 10260 SCEVWrapPredicate::IncrementWrapFlags 10261 SCEVWrapPredicate::getImpliedFlags(const SCEVAddRecExpr *AR, 10262 ScalarEvolution &SE) { 10263 IncrementWrapFlags ImpliedFlags = IncrementAnyWrap; 10264 SCEV::NoWrapFlags StaticFlags = AR->getNoWrapFlags(); 10265 10266 // We can safely transfer the NSW flag as NSSW. 10267 if (ScalarEvolution::setFlags(StaticFlags, SCEV::FlagNSW) == StaticFlags) 10268 ImpliedFlags = IncrementNSSW; 10269 10270 if (ScalarEvolution::setFlags(StaticFlags, SCEV::FlagNUW) == StaticFlags) { 10271 // If the increment is positive, the SCEV NUW flag will also imply the 10272 // WrapPredicate NUSW flag. 10273 if (const auto *Step = dyn_cast<SCEVConstant>(AR->getStepRecurrence(SE))) 10274 if (Step->getValue()->getValue().isNonNegative()) 10275 ImpliedFlags = setFlags(ImpliedFlags, IncrementNUSW); 10276 } 10277 10278 return ImpliedFlags; 10279 } 10280 10281 /// Union predicates don't get cached so create a dummy set ID for it. 10282 SCEVUnionPredicate::SCEVUnionPredicate() 10283 : SCEVPredicate(FoldingSetNodeIDRef(nullptr, 0), P_Union) {} 10284 10285 bool SCEVUnionPredicate::isAlwaysTrue() const { 10286 return all_of(Preds, 10287 [](const SCEVPredicate *I) { return I->isAlwaysTrue(); }); 10288 } 10289 10290 ArrayRef<const SCEVPredicate *> 10291 SCEVUnionPredicate::getPredicatesForExpr(const SCEV *Expr) { 10292 auto I = SCEVToPreds.find(Expr); 10293 if (I == SCEVToPreds.end()) 10294 return ArrayRef<const SCEVPredicate *>(); 10295 return I->second; 10296 } 10297 10298 bool SCEVUnionPredicate::implies(const SCEVPredicate *N) const { 10299 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(N)) 10300 return all_of(Set->Preds, 10301 [this](const SCEVPredicate *I) { return this->implies(I); }); 10302 10303 auto ScevPredsIt = SCEVToPreds.find(N->getExpr()); 10304 if (ScevPredsIt == SCEVToPreds.end()) 10305 return false; 10306 auto &SCEVPreds = ScevPredsIt->second; 10307 10308 return any_of(SCEVPreds, 10309 [N](const SCEVPredicate *I) { return I->implies(N); }); 10310 } 10311 10312 const SCEV *SCEVUnionPredicate::getExpr() const { return nullptr; } 10313 10314 void SCEVUnionPredicate::print(raw_ostream &OS, unsigned Depth) const { 10315 for (auto Pred : Preds) 10316 Pred->print(OS, Depth); 10317 } 10318 10319 void SCEVUnionPredicate::add(const SCEVPredicate *N) { 10320 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(N)) { 10321 for (auto Pred : Set->Preds) 10322 add(Pred); 10323 return; 10324 } 10325 10326 if (implies(N)) 10327 return; 10328 10329 const SCEV *Key = N->getExpr(); 10330 assert(Key && "Only SCEVUnionPredicate doesn't have an " 10331 " associated expression!"); 10332 10333 SCEVToPreds[Key].push_back(N); 10334 Preds.push_back(N); 10335 } 10336 10337 PredicatedScalarEvolution::PredicatedScalarEvolution(ScalarEvolution &SE, 10338 Loop &L) 10339 : SE(SE), L(L), Generation(0), BackedgeCount(nullptr) {} 10340 10341 const SCEV *PredicatedScalarEvolution::getSCEV(Value *V) { 10342 const SCEV *Expr = SE.getSCEV(V); 10343 RewriteEntry &Entry = RewriteMap[Expr]; 10344 10345 // If we already have an entry and the version matches, return it. 10346 if (Entry.second && Generation == Entry.first) 10347 return Entry.second; 10348 10349 // We found an entry but it's stale. Rewrite the stale entry 10350 // acording to the current predicate. 10351 if (Entry.second) 10352 Expr = Entry.second; 10353 10354 const SCEV *NewSCEV = SE.rewriteUsingPredicate(Expr, &L, Preds); 10355 Entry = {Generation, NewSCEV}; 10356 10357 return NewSCEV; 10358 } 10359 10360 const SCEV *PredicatedScalarEvolution::getBackedgeTakenCount() { 10361 if (!BackedgeCount) { 10362 SCEVUnionPredicate BackedgePred; 10363 BackedgeCount = SE.getPredicatedBackedgeTakenCount(&L, BackedgePred); 10364 addPredicate(BackedgePred); 10365 } 10366 return BackedgeCount; 10367 } 10368 10369 void PredicatedScalarEvolution::addPredicate(const SCEVPredicate &Pred) { 10370 if (Preds.implies(&Pred)) 10371 return; 10372 Preds.add(&Pred); 10373 updateGeneration(); 10374 } 10375 10376 const SCEVUnionPredicate &PredicatedScalarEvolution::getUnionPredicate() const { 10377 return Preds; 10378 } 10379 10380 void PredicatedScalarEvolution::updateGeneration() { 10381 // If the generation number wrapped recompute everything. 10382 if (++Generation == 0) { 10383 for (auto &II : RewriteMap) { 10384 const SCEV *Rewritten = II.second.second; 10385 II.second = {Generation, SE.rewriteUsingPredicate(Rewritten, &L, Preds)}; 10386 } 10387 } 10388 } 10389 10390 void PredicatedScalarEvolution::setNoOverflow( 10391 Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags) { 10392 const SCEV *Expr = getSCEV(V); 10393 const auto *AR = cast<SCEVAddRecExpr>(Expr); 10394 10395 auto ImpliedFlags = SCEVWrapPredicate::getImpliedFlags(AR, SE); 10396 10397 // Clear the statically implied flags. 10398 Flags = SCEVWrapPredicate::clearFlags(Flags, ImpliedFlags); 10399 addPredicate(*SE.getWrapPredicate(AR, Flags)); 10400 10401 auto II = FlagsMap.insert({V, Flags}); 10402 if (!II.second) 10403 II.first->second = SCEVWrapPredicate::setFlags(Flags, II.first->second); 10404 } 10405 10406 bool PredicatedScalarEvolution::hasNoOverflow( 10407 Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags) { 10408 const SCEV *Expr = getSCEV(V); 10409 const auto *AR = cast<SCEVAddRecExpr>(Expr); 10410 10411 Flags = SCEVWrapPredicate::clearFlags( 10412 Flags, SCEVWrapPredicate::getImpliedFlags(AR, SE)); 10413 10414 auto II = FlagsMap.find(V); 10415 10416 if (II != FlagsMap.end()) 10417 Flags = SCEVWrapPredicate::clearFlags(Flags, II->second); 10418 10419 return Flags == SCEVWrapPredicate::IncrementAnyWrap; 10420 } 10421 10422 const SCEVAddRecExpr *PredicatedScalarEvolution::getAsAddRec(Value *V) { 10423 const SCEV *Expr = this->getSCEV(V); 10424 auto *New = SE.convertSCEVToAddRecWithPredicates(Expr, &L, Preds); 10425 10426 if (!New) 10427 return nullptr; 10428 10429 updateGeneration(); 10430 RewriteMap[SE.getSCEV(V)] = {Generation, New}; 10431 return New; 10432 } 10433 10434 PredicatedScalarEvolution::PredicatedScalarEvolution( 10435 const PredicatedScalarEvolution &Init) 10436 : RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L), Preds(Init.Preds), 10437 Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) { 10438 for (const auto &I : Init.FlagsMap) 10439 FlagsMap.insert(I); 10440 } 10441 10442 void PredicatedScalarEvolution::print(raw_ostream &OS, unsigned Depth) const { 10443 // For each block. 10444 for (auto *BB : L.getBlocks()) 10445 for (auto &I : *BB) { 10446 if (!SE.isSCEVable(I.getType())) 10447 continue; 10448 10449 auto *Expr = SE.getSCEV(&I); 10450 auto II = RewriteMap.find(Expr); 10451 10452 if (II == RewriteMap.end()) 10453 continue; 10454 10455 // Don't print things that are not interesting. 10456 if (II->second.second == Expr) 10457 continue; 10458 10459 OS.indent(Depth) << "[PSE]" << I << ":\n"; 10460 OS.indent(Depth + 2) << *Expr << "\n"; 10461 OS.indent(Depth + 2) << "--> " << *II->second.second << "\n"; 10462 } 10463 } 10464