1 //===---------- SplitKit.cpp - Toolkit for splitting live ranges ----------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file contains the SplitAnalysis class as well as mutator functions for 11 // live range splitting. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "SplitKit.h" 16 #include "llvm/ADT/Statistic.h" 17 #include "llvm/CodeGen/LiveIntervalAnalysis.h" 18 #include "llvm/CodeGen/LiveRangeEdit.h" 19 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" 20 #include "llvm/CodeGen/MachineDominators.h" 21 #include "llvm/CodeGen/MachineInstrBuilder.h" 22 #include "llvm/CodeGen/MachineLoopInfo.h" 23 #include "llvm/CodeGen/MachineRegisterInfo.h" 24 #include "llvm/CodeGen/VirtRegMap.h" 25 #include "llvm/Support/Debug.h" 26 #include "llvm/Support/raw_ostream.h" 27 #include "llvm/Target/TargetInstrInfo.h" 28 #include "llvm/Target/TargetMachine.h" 29 30 using namespace llvm; 31 32 #define DEBUG_TYPE "regalloc" 33 34 STATISTIC(NumFinished, "Number of splits finished"); 35 STATISTIC(NumSimple, "Number of splits that were simple"); 36 STATISTIC(NumCopies, "Number of copies inserted for splitting"); 37 STATISTIC(NumRemats, "Number of rematerialized defs for splitting"); 38 STATISTIC(NumRepairs, "Number of invalid live ranges repaired"); 39 40 //===----------------------------------------------------------------------===// 41 // Last Insert Point Analysis 42 //===----------------------------------------------------------------------===// 43 44 InsertPointAnalysis::InsertPointAnalysis(const LiveIntervals &lis, 45 unsigned BBNum) 46 : LIS(lis), LastInsertPoint(BBNum) {} 47 48 SlotIndex 49 InsertPointAnalysis::computeLastInsertPoint(const LiveInterval &CurLI, 50 const MachineBasicBlock &MBB) { 51 unsigned Num = MBB.getNumber(); 52 std::pair<SlotIndex, SlotIndex> &LIP = LastInsertPoint[Num]; 53 SlotIndex MBBEnd = LIS.getMBBEndIdx(&MBB); 54 55 SmallVector<const MachineBasicBlock *, 1> EHPadSucessors; 56 for (const MachineBasicBlock *SMBB : MBB.successors()) 57 if (SMBB->isEHPad()) 58 EHPadSucessors.push_back(SMBB); 59 60 // Compute insert points on the first call. The pair is independent of the 61 // current live interval. 62 if (!LIP.first.isValid()) { 63 MachineBasicBlock::const_iterator FirstTerm = MBB.getFirstTerminator(); 64 if (FirstTerm == MBB.end()) 65 LIP.first = MBBEnd; 66 else 67 LIP.first = LIS.getInstructionIndex(*FirstTerm); 68 69 // If there is a landing pad successor, also find the call instruction. 70 if (EHPadSucessors.empty()) 71 return LIP.first; 72 // There may not be a call instruction (?) in which case we ignore LPad. 73 LIP.second = LIP.first; 74 for (MachineBasicBlock::const_iterator I = MBB.end(), E = MBB.begin(); 75 I != E;) { 76 --I; 77 if (I->isCall()) { 78 LIP.second = LIS.getInstructionIndex(*I); 79 break; 80 } 81 } 82 } 83 84 // If CurLI is live into a landing pad successor, move the last insert point 85 // back to the call that may throw. 86 if (!LIP.second) 87 return LIP.first; 88 89 if (none_of(EHPadSucessors, [&](const MachineBasicBlock *EHPad) { 90 return LIS.isLiveInToMBB(CurLI, EHPad); 91 })) 92 return LIP.first; 93 94 // Find the value leaving MBB. 95 const VNInfo *VNI = CurLI.getVNInfoBefore(MBBEnd); 96 if (!VNI) 97 return LIP.first; 98 99 // If the value leaving MBB was defined after the call in MBB, it can't 100 // really be live-in to the landing pad. This can happen if the landing pad 101 // has a PHI, and this register is undef on the exceptional edge. 102 // <rdar://problem/10664933> 103 if (!SlotIndex::isEarlierInstr(VNI->def, LIP.second) && VNI->def < MBBEnd) 104 return LIP.first; 105 106 // Value is properly live-in to the landing pad. 107 // Only allow inserts before the call. 108 return LIP.second; 109 } 110 111 MachineBasicBlock::iterator 112 InsertPointAnalysis::getLastInsertPointIter(const LiveInterval &CurLI, 113 MachineBasicBlock &MBB) { 114 SlotIndex LIP = getLastInsertPoint(CurLI, MBB); 115 if (LIP == LIS.getMBBEndIdx(&MBB)) 116 return MBB.end(); 117 return LIS.getInstructionFromIndex(LIP); 118 } 119 120 //===----------------------------------------------------------------------===// 121 // Split Analysis 122 //===----------------------------------------------------------------------===// 123 124 SplitAnalysis::SplitAnalysis(const VirtRegMap &vrm, const LiveIntervals &lis, 125 const MachineLoopInfo &mli) 126 : MF(vrm.getMachineFunction()), VRM(vrm), LIS(lis), Loops(mli), 127 TII(*MF.getSubtarget().getInstrInfo()), CurLI(nullptr), 128 IPA(lis, MF.getNumBlockIDs()) {} 129 130 void SplitAnalysis::clear() { 131 UseSlots.clear(); 132 UseBlocks.clear(); 133 ThroughBlocks.clear(); 134 CurLI = nullptr; 135 DidRepairRange = false; 136 } 137 138 /// analyzeUses - Count instructions, basic blocks, and loops using CurLI. 139 void SplitAnalysis::analyzeUses() { 140 assert(UseSlots.empty() && "Call clear first"); 141 142 // First get all the defs from the interval values. This provides the correct 143 // slots for early clobbers. 144 for (const VNInfo *VNI : CurLI->valnos) 145 if (!VNI->isPHIDef() && !VNI->isUnused()) 146 UseSlots.push_back(VNI->def); 147 148 // Get use slots form the use-def chain. 149 const MachineRegisterInfo &MRI = MF.getRegInfo(); 150 for (MachineOperand &MO : MRI.use_nodbg_operands(CurLI->reg)) 151 if (!MO.isUndef()) 152 UseSlots.push_back(LIS.getInstructionIndex(*MO.getParent()).getRegSlot()); 153 154 array_pod_sort(UseSlots.begin(), UseSlots.end()); 155 156 // Remove duplicates, keeping the smaller slot for each instruction. 157 // That is what we want for early clobbers. 158 UseSlots.erase(std::unique(UseSlots.begin(), UseSlots.end(), 159 SlotIndex::isSameInstr), 160 UseSlots.end()); 161 162 // Compute per-live block info. 163 if (!calcLiveBlockInfo()) { 164 // FIXME: calcLiveBlockInfo found inconsistencies in the live range. 165 // I am looking at you, RegisterCoalescer! 166 DidRepairRange = true; 167 ++NumRepairs; 168 DEBUG(dbgs() << "*** Fixing inconsistent live interval! ***\n"); 169 const_cast<LiveIntervals&>(LIS) 170 .shrinkToUses(const_cast<LiveInterval*>(CurLI)); 171 UseBlocks.clear(); 172 ThroughBlocks.clear(); 173 bool fixed = calcLiveBlockInfo(); 174 (void)fixed; 175 assert(fixed && "Couldn't fix broken live interval"); 176 } 177 178 DEBUG(dbgs() << "Analyze counted " 179 << UseSlots.size() << " instrs in " 180 << UseBlocks.size() << " blocks, through " 181 << NumThroughBlocks << " blocks.\n"); 182 } 183 184 /// calcLiveBlockInfo - Fill the LiveBlocks array with information about blocks 185 /// where CurLI is live. 186 bool SplitAnalysis::calcLiveBlockInfo() { 187 ThroughBlocks.resize(MF.getNumBlockIDs()); 188 NumThroughBlocks = NumGapBlocks = 0; 189 if (CurLI->empty()) 190 return true; 191 192 LiveInterval::const_iterator LVI = CurLI->begin(); 193 LiveInterval::const_iterator LVE = CurLI->end(); 194 195 SmallVectorImpl<SlotIndex>::const_iterator UseI, UseE; 196 UseI = UseSlots.begin(); 197 UseE = UseSlots.end(); 198 199 // Loop over basic blocks where CurLI is live. 200 MachineFunction::iterator MFI = 201 LIS.getMBBFromIndex(LVI->start)->getIterator(); 202 for (;;) { 203 BlockInfo BI; 204 BI.MBB = &*MFI; 205 SlotIndex Start, Stop; 206 std::tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB); 207 208 // If the block contains no uses, the range must be live through. At one 209 // point, RegisterCoalescer could create dangling ranges that ended 210 // mid-block. 211 if (UseI == UseE || *UseI >= Stop) { 212 ++NumThroughBlocks; 213 ThroughBlocks.set(BI.MBB->getNumber()); 214 // The range shouldn't end mid-block if there are no uses. This shouldn't 215 // happen. 216 if (LVI->end < Stop) 217 return false; 218 } else { 219 // This block has uses. Find the first and last uses in the block. 220 BI.FirstInstr = *UseI; 221 assert(BI.FirstInstr >= Start); 222 do ++UseI; 223 while (UseI != UseE && *UseI < Stop); 224 BI.LastInstr = UseI[-1]; 225 assert(BI.LastInstr < Stop); 226 227 // LVI is the first live segment overlapping MBB. 228 BI.LiveIn = LVI->start <= Start; 229 230 // When not live in, the first use should be a def. 231 if (!BI.LiveIn) { 232 assert(LVI->start == LVI->valno->def && "Dangling Segment start"); 233 assert(LVI->start == BI.FirstInstr && "First instr should be a def"); 234 BI.FirstDef = BI.FirstInstr; 235 } 236 237 // Look for gaps in the live range. 238 BI.LiveOut = true; 239 while (LVI->end < Stop) { 240 SlotIndex LastStop = LVI->end; 241 if (++LVI == LVE || LVI->start >= Stop) { 242 BI.LiveOut = false; 243 BI.LastInstr = LastStop; 244 break; 245 } 246 247 if (LastStop < LVI->start) { 248 // There is a gap in the live range. Create duplicate entries for the 249 // live-in snippet and the live-out snippet. 250 ++NumGapBlocks; 251 252 // Push the Live-in part. 253 BI.LiveOut = false; 254 UseBlocks.push_back(BI); 255 UseBlocks.back().LastInstr = LastStop; 256 257 // Set up BI for the live-out part. 258 BI.LiveIn = false; 259 BI.LiveOut = true; 260 BI.FirstInstr = BI.FirstDef = LVI->start; 261 } 262 263 // A Segment that starts in the middle of the block must be a def. 264 assert(LVI->start == LVI->valno->def && "Dangling Segment start"); 265 if (!BI.FirstDef) 266 BI.FirstDef = LVI->start; 267 } 268 269 UseBlocks.push_back(BI); 270 271 // LVI is now at LVE or LVI->end >= Stop. 272 if (LVI == LVE) 273 break; 274 } 275 276 // Live segment ends exactly at Stop. Move to the next segment. 277 if (LVI->end == Stop && ++LVI == LVE) 278 break; 279 280 // Pick the next basic block. 281 if (LVI->start < Stop) 282 ++MFI; 283 else 284 MFI = LIS.getMBBFromIndex(LVI->start)->getIterator(); 285 } 286 287 assert(getNumLiveBlocks() == countLiveBlocks(CurLI) && "Bad block count"); 288 return true; 289 } 290 291 unsigned SplitAnalysis::countLiveBlocks(const LiveInterval *cli) const { 292 if (cli->empty()) 293 return 0; 294 LiveInterval *li = const_cast<LiveInterval*>(cli); 295 LiveInterval::iterator LVI = li->begin(); 296 LiveInterval::iterator LVE = li->end(); 297 unsigned Count = 0; 298 299 // Loop over basic blocks where li is live. 300 MachineFunction::const_iterator MFI = 301 LIS.getMBBFromIndex(LVI->start)->getIterator(); 302 SlotIndex Stop = LIS.getMBBEndIdx(&*MFI); 303 for (;;) { 304 ++Count; 305 LVI = li->advanceTo(LVI, Stop); 306 if (LVI == LVE) 307 return Count; 308 do { 309 ++MFI; 310 Stop = LIS.getMBBEndIdx(&*MFI); 311 } while (Stop <= LVI->start); 312 } 313 } 314 315 bool SplitAnalysis::isOriginalEndpoint(SlotIndex Idx) const { 316 unsigned OrigReg = VRM.getOriginal(CurLI->reg); 317 const LiveInterval &Orig = LIS.getInterval(OrigReg); 318 assert(!Orig.empty() && "Splitting empty interval?"); 319 LiveInterval::const_iterator I = Orig.find(Idx); 320 321 // Range containing Idx should begin at Idx. 322 if (I != Orig.end() && I->start <= Idx) 323 return I->start == Idx; 324 325 // Range does not contain Idx, previous must end at Idx. 326 return I != Orig.begin() && (--I)->end == Idx; 327 } 328 329 void SplitAnalysis::analyze(const LiveInterval *li) { 330 clear(); 331 CurLI = li; 332 analyzeUses(); 333 } 334 335 336 //===----------------------------------------------------------------------===// 337 // Split Editor 338 //===----------------------------------------------------------------------===// 339 340 /// Create a new SplitEditor for editing the LiveInterval analyzed by SA. 341 SplitEditor::SplitEditor(SplitAnalysis &sa, AliasAnalysis &aa, 342 LiveIntervals &lis, VirtRegMap &vrm, 343 MachineDominatorTree &mdt, 344 MachineBlockFrequencyInfo &mbfi) 345 : SA(sa), AA(aa), LIS(lis), VRM(vrm), 346 MRI(vrm.getMachineFunction().getRegInfo()), MDT(mdt), 347 TII(*vrm.getMachineFunction().getSubtarget().getInstrInfo()), 348 TRI(*vrm.getMachineFunction().getSubtarget().getRegisterInfo()), 349 MBFI(mbfi), Edit(nullptr), OpenIdx(0), SpillMode(SM_Partition), 350 RegAssign(Allocator) {} 351 352 void SplitEditor::reset(LiveRangeEdit &LRE, ComplementSpillMode SM) { 353 Edit = &LRE; 354 SpillMode = SM; 355 OpenIdx = 0; 356 RegAssign.clear(); 357 Values.clear(); 358 359 // Reset the LiveRangeCalc instances needed for this spill mode. 360 LRCalc[0].reset(&VRM.getMachineFunction(), LIS.getSlotIndexes(), &MDT, 361 &LIS.getVNInfoAllocator()); 362 if (SpillMode) 363 LRCalc[1].reset(&VRM.getMachineFunction(), LIS.getSlotIndexes(), &MDT, 364 &LIS.getVNInfoAllocator()); 365 366 // We don't need an AliasAnalysis since we will only be performing 367 // cheap-as-a-copy remats anyway. 368 Edit->anyRematerializable(nullptr); 369 } 370 371 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 372 LLVM_DUMP_METHOD void SplitEditor::dump() const { 373 if (RegAssign.empty()) { 374 dbgs() << " empty\n"; 375 return; 376 } 377 378 for (RegAssignMap::const_iterator I = RegAssign.begin(); I.valid(); ++I) 379 dbgs() << " [" << I.start() << ';' << I.stop() << "):" << I.value(); 380 dbgs() << '\n'; 381 } 382 #endif 383 384 VNInfo *SplitEditor::defValue(unsigned RegIdx, 385 const VNInfo *ParentVNI, 386 SlotIndex Idx) { 387 assert(ParentVNI && "Mapping NULL value"); 388 assert(Idx.isValid() && "Invalid SlotIndex"); 389 assert(Edit->getParent().getVNInfoAt(Idx) == ParentVNI && "Bad Parent VNI"); 390 LiveInterval *LI = &LIS.getInterval(Edit->get(RegIdx)); 391 392 // Create a new value. 393 VNInfo *VNI = LI->getNextValue(Idx, LIS.getVNInfoAllocator()); 394 395 // Use insert for lookup, so we can add missing values with a second lookup. 396 std::pair<ValueMap::iterator, bool> InsP = 397 Values.insert(std::make_pair(std::make_pair(RegIdx, ParentVNI->id), 398 ValueForcePair(VNI, false))); 399 400 // This was the first time (RegIdx, ParentVNI) was mapped. 401 // Keep it as a simple def without any liveness. 402 if (InsP.second) 403 return VNI; 404 405 // If the previous value was a simple mapping, add liveness for it now. 406 if (VNInfo *OldVNI = InsP.first->second.getPointer()) { 407 SlotIndex Def = OldVNI->def; 408 LI->addSegment(LiveInterval::Segment(Def, Def.getDeadSlot(), OldVNI)); 409 // No longer a simple mapping. Switch to a complex, non-forced mapping. 410 InsP.first->second = ValueForcePair(); 411 } 412 413 // This is a complex mapping, add liveness for VNI 414 SlotIndex Def = VNI->def; 415 LI->addSegment(LiveInterval::Segment(Def, Def.getDeadSlot(), VNI)); 416 417 return VNI; 418 } 419 420 void SplitEditor::forceRecompute(unsigned RegIdx, const VNInfo *ParentVNI) { 421 assert(ParentVNI && "Mapping NULL value"); 422 ValueForcePair &VFP = Values[std::make_pair(RegIdx, ParentVNI->id)]; 423 VNInfo *VNI = VFP.getPointer(); 424 425 // ParentVNI was either unmapped or already complex mapped. Either way, just 426 // set the force bit. 427 if (!VNI) { 428 VFP.setInt(true); 429 return; 430 } 431 432 // This was previously a single mapping. Make sure the old def is represented 433 // by a trivial live range. 434 SlotIndex Def = VNI->def; 435 LiveInterval *LI = &LIS.getInterval(Edit->get(RegIdx)); 436 LI->addSegment(LiveInterval::Segment(Def, Def.getDeadSlot(), VNI)); 437 // Mark as complex mapped, forced. 438 VFP = ValueForcePair(nullptr, true); 439 } 440 441 VNInfo *SplitEditor::defFromParent(unsigned RegIdx, 442 VNInfo *ParentVNI, 443 SlotIndex UseIdx, 444 MachineBasicBlock &MBB, 445 MachineBasicBlock::iterator I) { 446 MachineInstr *CopyMI = nullptr; 447 SlotIndex Def; 448 LiveInterval *LI = &LIS.getInterval(Edit->get(RegIdx)); 449 450 // We may be trying to avoid interference that ends at a deleted instruction, 451 // so always begin RegIdx 0 early and all others late. 452 bool Late = RegIdx != 0; 453 454 // Attempt cheap-as-a-copy rematerialization. 455 unsigned Original = VRM.getOriginal(Edit->get(RegIdx)); 456 LiveInterval &OrigLI = LIS.getInterval(Original); 457 VNInfo *OrigVNI = OrigLI.getVNInfoAt(UseIdx); 458 LiveRangeEdit::Remat RM(ParentVNI); 459 RM.OrigMI = LIS.getInstructionFromIndex(OrigVNI->def); 460 461 if (Edit->canRematerializeAt(RM, OrigVNI, UseIdx, true)) { 462 Def = Edit->rematerializeAt(MBB, I, LI->reg, RM, TRI, Late); 463 ++NumRemats; 464 } else { 465 // Can't remat, just insert a copy from parent. 466 CopyMI = BuildMI(MBB, I, DebugLoc(), TII.get(TargetOpcode::COPY), LI->reg) 467 .addReg(Edit->getReg()); 468 Def = LIS.getSlotIndexes() 469 ->insertMachineInstrInMaps(*CopyMI, Late) 470 .getRegSlot(); 471 ++NumCopies; 472 } 473 474 // Define the value in Reg. 475 return defValue(RegIdx, ParentVNI, Def); 476 } 477 478 /// Create a new virtual register and live interval. 479 unsigned SplitEditor::openIntv() { 480 // Create the complement as index 0. 481 if (Edit->empty()) 482 Edit->createEmptyInterval(); 483 484 // Create the open interval. 485 OpenIdx = Edit->size(); 486 Edit->createEmptyInterval(); 487 return OpenIdx; 488 } 489 490 void SplitEditor::selectIntv(unsigned Idx) { 491 assert(Idx != 0 && "Cannot select the complement interval"); 492 assert(Idx < Edit->size() && "Can only select previously opened interval"); 493 DEBUG(dbgs() << " selectIntv " << OpenIdx << " -> " << Idx << '\n'); 494 OpenIdx = Idx; 495 } 496 497 SlotIndex SplitEditor::enterIntvBefore(SlotIndex Idx) { 498 assert(OpenIdx && "openIntv not called before enterIntvBefore"); 499 DEBUG(dbgs() << " enterIntvBefore " << Idx); 500 Idx = Idx.getBaseIndex(); 501 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx); 502 if (!ParentVNI) { 503 DEBUG(dbgs() << ": not live\n"); 504 return Idx; 505 } 506 DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n'); 507 MachineInstr *MI = LIS.getInstructionFromIndex(Idx); 508 assert(MI && "enterIntvBefore called with invalid index"); 509 510 VNInfo *VNI = defFromParent(OpenIdx, ParentVNI, Idx, *MI->getParent(), MI); 511 return VNI->def; 512 } 513 514 SlotIndex SplitEditor::enterIntvAfter(SlotIndex Idx) { 515 assert(OpenIdx && "openIntv not called before enterIntvAfter"); 516 DEBUG(dbgs() << " enterIntvAfter " << Idx); 517 Idx = Idx.getBoundaryIndex(); 518 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx); 519 if (!ParentVNI) { 520 DEBUG(dbgs() << ": not live\n"); 521 return Idx; 522 } 523 DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n'); 524 MachineInstr *MI = LIS.getInstructionFromIndex(Idx); 525 assert(MI && "enterIntvAfter called with invalid index"); 526 527 VNInfo *VNI = defFromParent(OpenIdx, ParentVNI, Idx, *MI->getParent(), 528 std::next(MachineBasicBlock::iterator(MI))); 529 return VNI->def; 530 } 531 532 SlotIndex SplitEditor::enterIntvAtEnd(MachineBasicBlock &MBB) { 533 assert(OpenIdx && "openIntv not called before enterIntvAtEnd"); 534 SlotIndex End = LIS.getMBBEndIdx(&MBB); 535 SlotIndex Last = End.getPrevSlot(); 536 DEBUG(dbgs() << " enterIntvAtEnd BB#" << MBB.getNumber() << ", " << Last); 537 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Last); 538 if (!ParentVNI) { 539 DEBUG(dbgs() << ": not live\n"); 540 return End; 541 } 542 DEBUG(dbgs() << ": valno " << ParentVNI->id); 543 VNInfo *VNI = defFromParent(OpenIdx, ParentVNI, Last, MBB, 544 SA.getLastSplitPointIter(&MBB)); 545 RegAssign.insert(VNI->def, End, OpenIdx); 546 DEBUG(dump()); 547 return VNI->def; 548 } 549 550 /// useIntv - indicate that all instructions in MBB should use OpenLI. 551 void SplitEditor::useIntv(const MachineBasicBlock &MBB) { 552 useIntv(LIS.getMBBStartIdx(&MBB), LIS.getMBBEndIdx(&MBB)); 553 } 554 555 void SplitEditor::useIntv(SlotIndex Start, SlotIndex End) { 556 assert(OpenIdx && "openIntv not called before useIntv"); 557 DEBUG(dbgs() << " useIntv [" << Start << ';' << End << "):"); 558 RegAssign.insert(Start, End, OpenIdx); 559 DEBUG(dump()); 560 } 561 562 SlotIndex SplitEditor::leaveIntvAfter(SlotIndex Idx) { 563 assert(OpenIdx && "openIntv not called before leaveIntvAfter"); 564 DEBUG(dbgs() << " leaveIntvAfter " << Idx); 565 566 // The interval must be live beyond the instruction at Idx. 567 SlotIndex Boundary = Idx.getBoundaryIndex(); 568 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Boundary); 569 if (!ParentVNI) { 570 DEBUG(dbgs() << ": not live\n"); 571 return Boundary.getNextSlot(); 572 } 573 DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n'); 574 MachineInstr *MI = LIS.getInstructionFromIndex(Boundary); 575 assert(MI && "No instruction at index"); 576 577 // In spill mode, make live ranges as short as possible by inserting the copy 578 // before MI. This is only possible if that instruction doesn't redefine the 579 // value. The inserted COPY is not a kill, and we don't need to recompute 580 // the source live range. The spiller also won't try to hoist this copy. 581 if (SpillMode && !SlotIndex::isSameInstr(ParentVNI->def, Idx) && 582 MI->readsVirtualRegister(Edit->getReg())) { 583 forceRecompute(0, ParentVNI); 584 defFromParent(0, ParentVNI, Idx, *MI->getParent(), MI); 585 return Idx; 586 } 587 588 VNInfo *VNI = defFromParent(0, ParentVNI, Boundary, *MI->getParent(), 589 std::next(MachineBasicBlock::iterator(MI))); 590 return VNI->def; 591 } 592 593 SlotIndex SplitEditor::leaveIntvBefore(SlotIndex Idx) { 594 assert(OpenIdx && "openIntv not called before leaveIntvBefore"); 595 DEBUG(dbgs() << " leaveIntvBefore " << Idx); 596 597 // The interval must be live into the instruction at Idx. 598 Idx = Idx.getBaseIndex(); 599 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx); 600 if (!ParentVNI) { 601 DEBUG(dbgs() << ": not live\n"); 602 return Idx.getNextSlot(); 603 } 604 DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n'); 605 606 MachineInstr *MI = LIS.getInstructionFromIndex(Idx); 607 assert(MI && "No instruction at index"); 608 VNInfo *VNI = defFromParent(0, ParentVNI, Idx, *MI->getParent(), MI); 609 return VNI->def; 610 } 611 612 SlotIndex SplitEditor::leaveIntvAtTop(MachineBasicBlock &MBB) { 613 assert(OpenIdx && "openIntv not called before leaveIntvAtTop"); 614 SlotIndex Start = LIS.getMBBStartIdx(&MBB); 615 DEBUG(dbgs() << " leaveIntvAtTop BB#" << MBB.getNumber() << ", " << Start); 616 617 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Start); 618 if (!ParentVNI) { 619 DEBUG(dbgs() << ": not live\n"); 620 return Start; 621 } 622 623 VNInfo *VNI = defFromParent(0, ParentVNI, Start, MBB, 624 MBB.SkipPHIsAndLabels(MBB.begin())); 625 RegAssign.insert(Start, VNI->def, OpenIdx); 626 DEBUG(dump()); 627 return VNI->def; 628 } 629 630 void SplitEditor::overlapIntv(SlotIndex Start, SlotIndex End) { 631 assert(OpenIdx && "openIntv not called before overlapIntv"); 632 const VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Start); 633 assert(ParentVNI == Edit->getParent().getVNInfoBefore(End) && 634 "Parent changes value in extended range"); 635 assert(LIS.getMBBFromIndex(Start) == LIS.getMBBFromIndex(End) && 636 "Range cannot span basic blocks"); 637 638 // The complement interval will be extended as needed by LRCalc.extend(). 639 if (ParentVNI) 640 forceRecompute(0, ParentVNI); 641 DEBUG(dbgs() << " overlapIntv [" << Start << ';' << End << "):"); 642 RegAssign.insert(Start, End, OpenIdx); 643 DEBUG(dump()); 644 } 645 646 //===----------------------------------------------------------------------===// 647 // Spill modes 648 //===----------------------------------------------------------------------===// 649 650 void SplitEditor::removeBackCopies(SmallVectorImpl<VNInfo*> &Copies) { 651 LiveInterval *LI = &LIS.getInterval(Edit->get(0)); 652 DEBUG(dbgs() << "Removing " << Copies.size() << " back-copies.\n"); 653 RegAssignMap::iterator AssignI; 654 AssignI.setMap(RegAssign); 655 656 for (unsigned i = 0, e = Copies.size(); i != e; ++i) { 657 SlotIndex Def = Copies[i]->def; 658 MachineInstr *MI = LIS.getInstructionFromIndex(Def); 659 assert(MI && "No instruction for back-copy"); 660 661 MachineBasicBlock *MBB = MI->getParent(); 662 MachineBasicBlock::iterator MBBI(MI); 663 bool AtBegin; 664 do AtBegin = MBBI == MBB->begin(); 665 while (!AtBegin && (--MBBI)->isDebugValue()); 666 667 DEBUG(dbgs() << "Removing " << Def << '\t' << *MI); 668 LIS.removeVRegDefAt(*LI, Def); 669 LIS.RemoveMachineInstrFromMaps(*MI); 670 MI->eraseFromParent(); 671 672 // Adjust RegAssign if a register assignment is killed at Def. We want to 673 // avoid calculating the live range of the source register if possible. 674 AssignI.find(Def.getPrevSlot()); 675 if (!AssignI.valid() || AssignI.start() >= Def) 676 continue; 677 // If MI doesn't kill the assigned register, just leave it. 678 if (AssignI.stop() != Def) 679 continue; 680 unsigned RegIdx = AssignI.value(); 681 if (AtBegin || !MBBI->readsVirtualRegister(Edit->getReg())) { 682 DEBUG(dbgs() << " cannot find simple kill of RegIdx " << RegIdx << '\n'); 683 forceRecompute(RegIdx, Edit->getParent().getVNInfoAt(Def)); 684 } else { 685 SlotIndex Kill = LIS.getInstructionIndex(*MBBI).getRegSlot(); 686 DEBUG(dbgs() << " move kill to " << Kill << '\t' << *MBBI); 687 AssignI.setStop(Kill); 688 } 689 } 690 } 691 692 MachineBasicBlock* 693 SplitEditor::findShallowDominator(MachineBasicBlock *MBB, 694 MachineBasicBlock *DefMBB) { 695 if (MBB == DefMBB) 696 return MBB; 697 assert(MDT.dominates(DefMBB, MBB) && "MBB must be dominated by the def."); 698 699 const MachineLoopInfo &Loops = SA.Loops; 700 const MachineLoop *DefLoop = Loops.getLoopFor(DefMBB); 701 MachineDomTreeNode *DefDomNode = MDT[DefMBB]; 702 703 // Best candidate so far. 704 MachineBasicBlock *BestMBB = MBB; 705 unsigned BestDepth = UINT_MAX; 706 707 for (;;) { 708 const MachineLoop *Loop = Loops.getLoopFor(MBB); 709 710 // MBB isn't in a loop, it doesn't get any better. All dominators have a 711 // higher frequency by definition. 712 if (!Loop) { 713 DEBUG(dbgs() << "Def in BB#" << DefMBB->getNumber() << " dominates BB#" 714 << MBB->getNumber() << " at depth 0\n"); 715 return MBB; 716 } 717 718 // We'll never be able to exit the DefLoop. 719 if (Loop == DefLoop) { 720 DEBUG(dbgs() << "Def in BB#" << DefMBB->getNumber() << " dominates BB#" 721 << MBB->getNumber() << " in the same loop\n"); 722 return MBB; 723 } 724 725 // Least busy dominator seen so far. 726 unsigned Depth = Loop->getLoopDepth(); 727 if (Depth < BestDepth) { 728 BestMBB = MBB; 729 BestDepth = Depth; 730 DEBUG(dbgs() << "Def in BB#" << DefMBB->getNumber() << " dominates BB#" 731 << MBB->getNumber() << " at depth " << Depth << '\n'); 732 } 733 734 // Leave loop by going to the immediate dominator of the loop header. 735 // This is a bigger stride than simply walking up the dominator tree. 736 MachineDomTreeNode *IDom = MDT[Loop->getHeader()]->getIDom(); 737 738 // Too far up the dominator tree? 739 if (!IDom || !MDT.dominates(DefDomNode, IDom)) 740 return BestMBB; 741 742 MBB = IDom->getBlock(); 743 } 744 } 745 746 void SplitEditor::computeRedundantBackCopies( 747 DenseSet<unsigned> &NotToHoistSet, SmallVectorImpl<VNInfo *> &BackCopies) { 748 LiveInterval *LI = &LIS.getInterval(Edit->get(0)); 749 LiveInterval *Parent = &Edit->getParent(); 750 SmallVector<SmallPtrSet<VNInfo *, 8>, 8> EqualVNs(Parent->getNumValNums()); 751 SmallPtrSet<VNInfo *, 8> DominatedVNIs; 752 753 // Aggregate VNIs having the same value as ParentVNI. 754 for (VNInfo *VNI : LI->valnos) { 755 if (VNI->isUnused()) 756 continue; 757 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(VNI->def); 758 EqualVNs[ParentVNI->id].insert(VNI); 759 } 760 761 // For VNI aggregation of each ParentVNI, collect dominated, i.e., 762 // redundant VNIs to BackCopies. 763 for (unsigned i = 0, e = Parent->getNumValNums(); i != e; ++i) { 764 VNInfo *ParentVNI = Parent->getValNumInfo(i); 765 if (!NotToHoistSet.count(ParentVNI->id)) 766 continue; 767 SmallPtrSetIterator<VNInfo *> It1 = EqualVNs[ParentVNI->id].begin(); 768 SmallPtrSetIterator<VNInfo *> It2 = It1; 769 for (; It1 != EqualVNs[ParentVNI->id].end(); ++It1) { 770 It2 = It1; 771 for (++It2; It2 != EqualVNs[ParentVNI->id].end(); ++It2) { 772 if (DominatedVNIs.count(*It1) || DominatedVNIs.count(*It2)) 773 continue; 774 775 MachineBasicBlock *MBB1 = LIS.getMBBFromIndex((*It1)->def); 776 MachineBasicBlock *MBB2 = LIS.getMBBFromIndex((*It2)->def); 777 if (MBB1 == MBB2) { 778 DominatedVNIs.insert((*It1)->def < (*It2)->def ? (*It2) : (*It1)); 779 } else if (MDT.dominates(MBB1, MBB2)) { 780 DominatedVNIs.insert(*It2); 781 } else if (MDT.dominates(MBB2, MBB1)) { 782 DominatedVNIs.insert(*It1); 783 } 784 } 785 } 786 if (!DominatedVNIs.empty()) { 787 forceRecompute(0, ParentVNI); 788 for (auto VNI : DominatedVNIs) { 789 BackCopies.push_back(VNI); 790 } 791 DominatedVNIs.clear(); 792 } 793 } 794 } 795 796 /// For SM_Size mode, find a common dominator for all the back-copies for 797 /// the same ParentVNI and hoist the backcopies to the dominator BB. 798 /// For SM_Speed mode, if the common dominator is hot and it is not beneficial 799 /// to do the hoisting, simply remove the dominated backcopies for the same 800 /// ParentVNI. 801 void SplitEditor::hoistCopies() { 802 // Get the complement interval, always RegIdx 0. 803 LiveInterval *LI = &LIS.getInterval(Edit->get(0)); 804 LiveInterval *Parent = &Edit->getParent(); 805 806 // Track the nearest common dominator for all back-copies for each ParentVNI, 807 // indexed by ParentVNI->id. 808 typedef std::pair<MachineBasicBlock*, SlotIndex> DomPair; 809 SmallVector<DomPair, 8> NearestDom(Parent->getNumValNums()); 810 // The total cost of all the back-copies for each ParentVNI. 811 SmallVector<BlockFrequency, 8> Costs(Parent->getNumValNums()); 812 // The ParentVNI->id set for which hoisting back-copies are not beneficial 813 // for Speed. 814 DenseSet<unsigned> NotToHoistSet; 815 816 // Find the nearest common dominator for parent values with multiple 817 // back-copies. If a single back-copy dominates, put it in DomPair.second. 818 for (VNInfo *VNI : LI->valnos) { 819 if (VNI->isUnused()) 820 continue; 821 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(VNI->def); 822 assert(ParentVNI && "Parent not live at complement def"); 823 824 // Don't hoist remats. The complement is probably going to disappear 825 // completely anyway. 826 if (Edit->didRematerialize(ParentVNI)) 827 continue; 828 829 MachineBasicBlock *ValMBB = LIS.getMBBFromIndex(VNI->def); 830 831 DomPair &Dom = NearestDom[ParentVNI->id]; 832 833 // Keep directly defined parent values. This is either a PHI or an 834 // instruction in the complement range. All other copies of ParentVNI 835 // should be eliminated. 836 if (VNI->def == ParentVNI->def) { 837 DEBUG(dbgs() << "Direct complement def at " << VNI->def << '\n'); 838 Dom = DomPair(ValMBB, VNI->def); 839 continue; 840 } 841 // Skip the singly mapped values. There is nothing to gain from hoisting a 842 // single back-copy. 843 if (Values.lookup(std::make_pair(0, ParentVNI->id)).getPointer()) { 844 DEBUG(dbgs() << "Single complement def at " << VNI->def << '\n'); 845 continue; 846 } 847 848 if (!Dom.first) { 849 // First time we see ParentVNI. VNI dominates itself. 850 Dom = DomPair(ValMBB, VNI->def); 851 } else if (Dom.first == ValMBB) { 852 // Two defs in the same block. Pick the earlier def. 853 if (!Dom.second.isValid() || VNI->def < Dom.second) 854 Dom.second = VNI->def; 855 } else { 856 // Different basic blocks. Check if one dominates. 857 MachineBasicBlock *Near = 858 MDT.findNearestCommonDominator(Dom.first, ValMBB); 859 if (Near == ValMBB) 860 // Def ValMBB dominates. 861 Dom = DomPair(ValMBB, VNI->def); 862 else if (Near != Dom.first) 863 // None dominate. Hoist to common dominator, need new def. 864 Dom = DomPair(Near, SlotIndex()); 865 Costs[ParentVNI->id] += MBFI.getBlockFreq(ValMBB); 866 } 867 868 DEBUG(dbgs() << "Multi-mapped complement " << VNI->id << '@' << VNI->def 869 << " for parent " << ParentVNI->id << '@' << ParentVNI->def 870 << " hoist to BB#" << Dom.first->getNumber() << ' ' 871 << Dom.second << '\n'); 872 } 873 874 // Insert the hoisted copies. 875 for (unsigned i = 0, e = Parent->getNumValNums(); i != e; ++i) { 876 DomPair &Dom = NearestDom[i]; 877 if (!Dom.first || Dom.second.isValid()) 878 continue; 879 // This value needs a hoisted copy inserted at the end of Dom.first. 880 VNInfo *ParentVNI = Parent->getValNumInfo(i); 881 MachineBasicBlock *DefMBB = LIS.getMBBFromIndex(ParentVNI->def); 882 // Get a less loopy dominator than Dom.first. 883 Dom.first = findShallowDominator(Dom.first, DefMBB); 884 if (SpillMode == SM_Speed && 885 MBFI.getBlockFreq(Dom.first) > Costs[ParentVNI->id]) { 886 NotToHoistSet.insert(ParentVNI->id); 887 continue; 888 } 889 SlotIndex Last = LIS.getMBBEndIdx(Dom.first).getPrevSlot(); 890 Dom.second = 891 defFromParent(0, ParentVNI, Last, *Dom.first, 892 SA.getLastSplitPointIter(Dom.first))->def; 893 } 894 895 // Remove redundant back-copies that are now known to be dominated by another 896 // def with the same value. 897 SmallVector<VNInfo*, 8> BackCopies; 898 for (VNInfo *VNI : LI->valnos) { 899 if (VNI->isUnused()) 900 continue; 901 VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(VNI->def); 902 const DomPair &Dom = NearestDom[ParentVNI->id]; 903 if (!Dom.first || Dom.second == VNI->def || 904 NotToHoistSet.count(ParentVNI->id)) 905 continue; 906 BackCopies.push_back(VNI); 907 forceRecompute(0, ParentVNI); 908 } 909 910 // If it is not beneficial to hoist all the BackCopies, simply remove 911 // redundant BackCopies in speed mode. 912 if (SpillMode == SM_Speed && !NotToHoistSet.empty()) 913 computeRedundantBackCopies(NotToHoistSet, BackCopies); 914 915 removeBackCopies(BackCopies); 916 } 917 918 919 /// transferValues - Transfer all possible values to the new live ranges. 920 /// Values that were rematerialized are left alone, they need LRCalc.extend(). 921 bool SplitEditor::transferValues() { 922 bool Skipped = false; 923 RegAssignMap::const_iterator AssignI = RegAssign.begin(); 924 for (const LiveRange::Segment &S : Edit->getParent()) { 925 DEBUG(dbgs() << " blit " << S << ':'); 926 VNInfo *ParentVNI = S.valno; 927 // RegAssign has holes where RegIdx 0 should be used. 928 SlotIndex Start = S.start; 929 AssignI.advanceTo(Start); 930 do { 931 unsigned RegIdx; 932 SlotIndex End = S.end; 933 if (!AssignI.valid()) { 934 RegIdx = 0; 935 } else if (AssignI.start() <= Start) { 936 RegIdx = AssignI.value(); 937 if (AssignI.stop() < End) { 938 End = AssignI.stop(); 939 ++AssignI; 940 } 941 } else { 942 RegIdx = 0; 943 End = std::min(End, AssignI.start()); 944 } 945 946 // The interval [Start;End) is continuously mapped to RegIdx, ParentVNI. 947 DEBUG(dbgs() << " [" << Start << ';' << End << ")=" << RegIdx); 948 LiveRange &LR = LIS.getInterval(Edit->get(RegIdx)); 949 950 // Check for a simply defined value that can be blitted directly. 951 ValueForcePair VFP = Values.lookup(std::make_pair(RegIdx, ParentVNI->id)); 952 if (VNInfo *VNI = VFP.getPointer()) { 953 DEBUG(dbgs() << ':' << VNI->id); 954 LR.addSegment(LiveInterval::Segment(Start, End, VNI)); 955 Start = End; 956 continue; 957 } 958 959 // Skip values with forced recomputation. 960 if (VFP.getInt()) { 961 DEBUG(dbgs() << "(recalc)"); 962 Skipped = true; 963 Start = End; 964 continue; 965 } 966 967 LiveRangeCalc &LRC = getLRCalc(RegIdx); 968 969 // This value has multiple defs in RegIdx, but it wasn't rematerialized, 970 // so the live range is accurate. Add live-in blocks in [Start;End) to the 971 // LiveInBlocks. 972 MachineFunction::iterator MBB = LIS.getMBBFromIndex(Start)->getIterator(); 973 SlotIndex BlockStart, BlockEnd; 974 std::tie(BlockStart, BlockEnd) = LIS.getSlotIndexes()->getMBBRange(&*MBB); 975 976 // The first block may be live-in, or it may have its own def. 977 if (Start != BlockStart) { 978 VNInfo *VNI = LR.extendInBlock(BlockStart, std::min(BlockEnd, End)); 979 assert(VNI && "Missing def for complex mapped value"); 980 DEBUG(dbgs() << ':' << VNI->id << "*BB#" << MBB->getNumber()); 981 // MBB has its own def. Is it also live-out? 982 if (BlockEnd <= End) 983 LRC.setLiveOutValue(&*MBB, VNI); 984 985 // Skip to the next block for live-in. 986 ++MBB; 987 BlockStart = BlockEnd; 988 } 989 990 // Handle the live-in blocks covered by [Start;End). 991 assert(Start <= BlockStart && "Expected live-in block"); 992 while (BlockStart < End) { 993 DEBUG(dbgs() << ">BB#" << MBB->getNumber()); 994 BlockEnd = LIS.getMBBEndIdx(&*MBB); 995 if (BlockStart == ParentVNI->def) { 996 // This block has the def of a parent PHI, so it isn't live-in. 997 assert(ParentVNI->isPHIDef() && "Non-phi defined at block start?"); 998 VNInfo *VNI = LR.extendInBlock(BlockStart, std::min(BlockEnd, End)); 999 assert(VNI && "Missing def for complex mapped parent PHI"); 1000 if (End >= BlockEnd) 1001 LRC.setLiveOutValue(&*MBB, VNI); // Live-out as well. 1002 } else { 1003 // This block needs a live-in value. The last block covered may not 1004 // be live-out. 1005 if (End < BlockEnd) 1006 LRC.addLiveInBlock(LR, MDT[&*MBB], End); 1007 else { 1008 // Live-through, and we don't know the value. 1009 LRC.addLiveInBlock(LR, MDT[&*MBB]); 1010 LRC.setLiveOutValue(&*MBB, nullptr); 1011 } 1012 } 1013 BlockStart = BlockEnd; 1014 ++MBB; 1015 } 1016 Start = End; 1017 } while (Start != S.end); 1018 DEBUG(dbgs() << '\n'); 1019 } 1020 1021 LRCalc[0].calculateValues(); 1022 if (SpillMode) 1023 LRCalc[1].calculateValues(); 1024 1025 return Skipped; 1026 } 1027 1028 void SplitEditor::extendPHIKillRanges() { 1029 // Extend live ranges to be live-out for successor PHI values. 1030 for (const VNInfo *PHIVNI : Edit->getParent().valnos) { 1031 if (PHIVNI->isUnused() || !PHIVNI->isPHIDef()) 1032 continue; 1033 unsigned RegIdx = RegAssign.lookup(PHIVNI->def); 1034 LiveRange &LR = LIS.getInterval(Edit->get(RegIdx)); 1035 1036 // Check whether PHI is dead. 1037 const LiveRange::Segment *Segment = LR.getSegmentContaining(PHIVNI->def); 1038 assert(Segment != nullptr && "Missing segment for VNI"); 1039 if (Segment->end == PHIVNI->def.getDeadSlot()) { 1040 // This is a dead PHI. Remove it. 1041 LR.removeSegment(*Segment, true); 1042 continue; 1043 } 1044 1045 LiveRangeCalc &LRC = getLRCalc(RegIdx); 1046 MachineBasicBlock *MBB = LIS.getMBBFromIndex(PHIVNI->def); 1047 for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), 1048 PE = MBB->pred_end(); PI != PE; ++PI) { 1049 SlotIndex End = LIS.getMBBEndIdx(*PI); 1050 SlotIndex LastUse = End.getPrevSlot(); 1051 // The predecessor may not have a live-out value. That is OK, like an 1052 // undef PHI operand. 1053 if (Edit->getParent().liveAt(LastUse)) { 1054 assert(RegAssign.lookup(LastUse) == RegIdx && 1055 "Different register assignment in phi predecessor"); 1056 LRC.extend(LR, End); 1057 } 1058 } 1059 } 1060 } 1061 1062 /// rewriteAssigned - Rewrite all uses of Edit->getReg(). 1063 void SplitEditor::rewriteAssigned(bool ExtendRanges) { 1064 for (MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(Edit->getReg()), 1065 RE = MRI.reg_end(); RI != RE;) { 1066 MachineOperand &MO = *RI; 1067 MachineInstr *MI = MO.getParent(); 1068 ++RI; 1069 // LiveDebugVariables should have handled all DBG_VALUE instructions. 1070 if (MI->isDebugValue()) { 1071 DEBUG(dbgs() << "Zapping " << *MI); 1072 MO.setReg(0); 1073 continue; 1074 } 1075 1076 // <undef> operands don't really read the register, so it doesn't matter 1077 // which register we choose. When the use operand is tied to a def, we must 1078 // use the same register as the def, so just do that always. 1079 SlotIndex Idx = LIS.getInstructionIndex(*MI); 1080 if (MO.isDef() || MO.isUndef()) 1081 Idx = Idx.getRegSlot(MO.isEarlyClobber()); 1082 1083 // Rewrite to the mapped register at Idx. 1084 unsigned RegIdx = RegAssign.lookup(Idx); 1085 LiveInterval *LI = &LIS.getInterval(Edit->get(RegIdx)); 1086 MO.setReg(LI->reg); 1087 DEBUG(dbgs() << " rewr BB#" << MI->getParent()->getNumber() << '\t' 1088 << Idx << ':' << RegIdx << '\t' << *MI); 1089 1090 // Extend liveness to Idx if the instruction reads reg. 1091 if (!ExtendRanges || MO.isUndef()) 1092 continue; 1093 1094 // Skip instructions that don't read Reg. 1095 if (MO.isDef()) { 1096 if (!MO.getSubReg() && !MO.isEarlyClobber()) 1097 continue; 1098 // We may wan't to extend a live range for a partial redef, or for a use 1099 // tied to an early clobber. 1100 Idx = Idx.getPrevSlot(); 1101 if (!Edit->getParent().liveAt(Idx)) 1102 continue; 1103 } else 1104 Idx = Idx.getRegSlot(true); 1105 1106 getLRCalc(RegIdx).extend(*LI, Idx.getNextSlot()); 1107 } 1108 } 1109 1110 void SplitEditor::deleteRematVictims() { 1111 SmallVector<MachineInstr*, 8> Dead; 1112 for (LiveRangeEdit::iterator I = Edit->begin(), E = Edit->end(); I != E; ++I){ 1113 LiveInterval *LI = &LIS.getInterval(*I); 1114 for (const LiveRange::Segment &S : LI->segments) { 1115 // Dead defs end at the dead slot. 1116 if (S.end != S.valno->def.getDeadSlot()) 1117 continue; 1118 if (S.valno->isPHIDef()) 1119 continue; 1120 MachineInstr *MI = LIS.getInstructionFromIndex(S.valno->def); 1121 assert(MI && "Missing instruction for dead def"); 1122 MI->addRegisterDead(LI->reg, &TRI); 1123 1124 if (!MI->allDefsAreDead()) 1125 continue; 1126 1127 DEBUG(dbgs() << "All defs dead: " << *MI); 1128 Dead.push_back(MI); 1129 } 1130 } 1131 1132 if (Dead.empty()) 1133 return; 1134 1135 Edit->eliminateDeadDefs(Dead, None, &AA); 1136 } 1137 1138 void SplitEditor::finish(SmallVectorImpl<unsigned> *LRMap) { 1139 ++NumFinished; 1140 1141 // At this point, the live intervals in Edit contain VNInfos corresponding to 1142 // the inserted copies. 1143 1144 // Add the original defs from the parent interval. 1145 for (const VNInfo *ParentVNI : Edit->getParent().valnos) { 1146 if (ParentVNI->isUnused()) 1147 continue; 1148 unsigned RegIdx = RegAssign.lookup(ParentVNI->def); 1149 defValue(RegIdx, ParentVNI, ParentVNI->def); 1150 1151 // Force rematted values to be recomputed everywhere. 1152 // The new live ranges may be truncated. 1153 if (Edit->didRematerialize(ParentVNI)) 1154 for (unsigned i = 0, e = Edit->size(); i != e; ++i) 1155 forceRecompute(i, ParentVNI); 1156 } 1157 1158 // Hoist back-copies to the complement interval when in spill mode. 1159 switch (SpillMode) { 1160 case SM_Partition: 1161 // Leave all back-copies as is. 1162 break; 1163 case SM_Size: 1164 case SM_Speed: 1165 // hoistCopies will behave differently between size and speed. 1166 hoistCopies(); 1167 } 1168 1169 // Transfer the simply mapped values, check if any are skipped. 1170 bool Skipped = transferValues(); 1171 1172 // Rewrite virtual registers, possibly extending ranges. 1173 rewriteAssigned(Skipped); 1174 1175 if (Skipped) 1176 extendPHIKillRanges(); 1177 else 1178 ++NumSimple; 1179 1180 // Delete defs that were rematted everywhere. 1181 if (Skipped) 1182 deleteRematVictims(); 1183 1184 // Get rid of unused values and set phi-kill flags. 1185 for (LiveRangeEdit::iterator I = Edit->begin(), E = Edit->end(); I != E; ++I) { 1186 LiveInterval &LI = LIS.getInterval(*I); 1187 LI.RenumberValues(); 1188 } 1189 1190 // Provide a reverse mapping from original indices to Edit ranges. 1191 if (LRMap) { 1192 LRMap->clear(); 1193 for (unsigned i = 0, e = Edit->size(); i != e; ++i) 1194 LRMap->push_back(i); 1195 } 1196 1197 // Now check if any registers were separated into multiple components. 1198 ConnectedVNInfoEqClasses ConEQ(LIS); 1199 for (unsigned i = 0, e = Edit->size(); i != e; ++i) { 1200 // Don't use iterators, they are invalidated by create() below. 1201 unsigned VReg = Edit->get(i); 1202 LiveInterval &LI = LIS.getInterval(VReg); 1203 SmallVector<LiveInterval*, 8> SplitLIs; 1204 LIS.splitSeparateComponents(LI, SplitLIs); 1205 unsigned Original = VRM.getOriginal(VReg); 1206 for (LiveInterval *SplitLI : SplitLIs) 1207 VRM.setIsSplitFromReg(SplitLI->reg, Original); 1208 1209 // The new intervals all map back to i. 1210 if (LRMap) 1211 LRMap->resize(Edit->size(), i); 1212 } 1213 1214 // Calculate spill weight and allocation hints for new intervals. 1215 Edit->calculateRegClassAndHint(VRM.getMachineFunction(), SA.Loops, MBFI); 1216 1217 assert(!LRMap || LRMap->size() == Edit->size()); 1218 } 1219 1220 1221 //===----------------------------------------------------------------------===// 1222 // Single Block Splitting 1223 //===----------------------------------------------------------------------===// 1224 1225 bool SplitAnalysis::shouldSplitSingleBlock(const BlockInfo &BI, 1226 bool SingleInstrs) const { 1227 // Always split for multiple instructions. 1228 if (!BI.isOneInstr()) 1229 return true; 1230 // Don't split for single instructions unless explicitly requested. 1231 if (!SingleInstrs) 1232 return false; 1233 // Splitting a live-through range always makes progress. 1234 if (BI.LiveIn && BI.LiveOut) 1235 return true; 1236 // No point in isolating a copy. It has no register class constraints. 1237 if (LIS.getInstructionFromIndex(BI.FirstInstr)->isCopyLike()) 1238 return false; 1239 // Finally, don't isolate an end point that was created by earlier splits. 1240 return isOriginalEndpoint(BI.FirstInstr); 1241 } 1242 1243 void SplitEditor::splitSingleBlock(const SplitAnalysis::BlockInfo &BI) { 1244 openIntv(); 1245 SlotIndex LastSplitPoint = SA.getLastSplitPoint(BI.MBB->getNumber()); 1246 SlotIndex SegStart = enterIntvBefore(std::min(BI.FirstInstr, 1247 LastSplitPoint)); 1248 if (!BI.LiveOut || BI.LastInstr < LastSplitPoint) { 1249 useIntv(SegStart, leaveIntvAfter(BI.LastInstr)); 1250 } else { 1251 // The last use is after the last valid split point. 1252 SlotIndex SegStop = leaveIntvBefore(LastSplitPoint); 1253 useIntv(SegStart, SegStop); 1254 overlapIntv(SegStop, BI.LastInstr); 1255 } 1256 } 1257 1258 1259 //===----------------------------------------------------------------------===// 1260 // Global Live Range Splitting Support 1261 //===----------------------------------------------------------------------===// 1262 1263 // These methods support a method of global live range splitting that uses a 1264 // global algorithm to decide intervals for CFG edges. They will insert split 1265 // points and color intervals in basic blocks while avoiding interference. 1266 // 1267 // Note that splitSingleBlock is also useful for blocks where both CFG edges 1268 // are on the stack. 1269 1270 void SplitEditor::splitLiveThroughBlock(unsigned MBBNum, 1271 unsigned IntvIn, SlotIndex LeaveBefore, 1272 unsigned IntvOut, SlotIndex EnterAfter){ 1273 SlotIndex Start, Stop; 1274 std::tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(MBBNum); 1275 1276 DEBUG(dbgs() << "BB#" << MBBNum << " [" << Start << ';' << Stop 1277 << ") intf " << LeaveBefore << '-' << EnterAfter 1278 << ", live-through " << IntvIn << " -> " << IntvOut); 1279 1280 assert((IntvIn || IntvOut) && "Use splitSingleBlock for isolated blocks"); 1281 1282 assert((!LeaveBefore || LeaveBefore < Stop) && "Interference after block"); 1283 assert((!IntvIn || !LeaveBefore || LeaveBefore > Start) && "Impossible intf"); 1284 assert((!EnterAfter || EnterAfter >= Start) && "Interference before block"); 1285 1286 MachineBasicBlock *MBB = VRM.getMachineFunction().getBlockNumbered(MBBNum); 1287 1288 if (!IntvOut) { 1289 DEBUG(dbgs() << ", spill on entry.\n"); 1290 // 1291 // <<<<<<<<< Possible LeaveBefore interference. 1292 // |-----------| Live through. 1293 // -____________ Spill on entry. 1294 // 1295 selectIntv(IntvIn); 1296 SlotIndex Idx = leaveIntvAtTop(*MBB); 1297 assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); 1298 (void)Idx; 1299 return; 1300 } 1301 1302 if (!IntvIn) { 1303 DEBUG(dbgs() << ", reload on exit.\n"); 1304 // 1305 // >>>>>>> Possible EnterAfter interference. 1306 // |-----------| Live through. 1307 // ___________-- Reload on exit. 1308 // 1309 selectIntv(IntvOut); 1310 SlotIndex Idx = enterIntvAtEnd(*MBB); 1311 assert((!EnterAfter || Idx >= EnterAfter) && "Interference"); 1312 (void)Idx; 1313 return; 1314 } 1315 1316 if (IntvIn == IntvOut && !LeaveBefore && !EnterAfter) { 1317 DEBUG(dbgs() << ", straight through.\n"); 1318 // 1319 // |-----------| Live through. 1320 // ------------- Straight through, same intv, no interference. 1321 // 1322 selectIntv(IntvOut); 1323 useIntv(Start, Stop); 1324 return; 1325 } 1326 1327 // We cannot legally insert splits after LSP. 1328 SlotIndex LSP = SA.getLastSplitPoint(MBBNum); 1329 assert((!IntvOut || !EnterAfter || EnterAfter < LSP) && "Impossible intf"); 1330 1331 if (IntvIn != IntvOut && (!LeaveBefore || !EnterAfter || 1332 LeaveBefore.getBaseIndex() > EnterAfter.getBoundaryIndex())) { 1333 DEBUG(dbgs() << ", switch avoiding interference.\n"); 1334 // 1335 // >>>> <<<< Non-overlapping EnterAfter/LeaveBefore interference. 1336 // |-----------| Live through. 1337 // ------======= Switch intervals between interference. 1338 // 1339 selectIntv(IntvOut); 1340 SlotIndex Idx; 1341 if (LeaveBefore && LeaveBefore < LSP) { 1342 Idx = enterIntvBefore(LeaveBefore); 1343 useIntv(Idx, Stop); 1344 } else { 1345 Idx = enterIntvAtEnd(*MBB); 1346 } 1347 selectIntv(IntvIn); 1348 useIntv(Start, Idx); 1349 assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); 1350 assert((!EnterAfter || Idx >= EnterAfter) && "Interference"); 1351 return; 1352 } 1353 1354 DEBUG(dbgs() << ", create local intv for interference.\n"); 1355 // 1356 // >>><><><><<<< Overlapping EnterAfter/LeaveBefore interference. 1357 // |-----------| Live through. 1358 // ==---------== Switch intervals before/after interference. 1359 // 1360 assert(LeaveBefore <= EnterAfter && "Missed case"); 1361 1362 selectIntv(IntvOut); 1363 SlotIndex Idx = enterIntvAfter(EnterAfter); 1364 useIntv(Idx, Stop); 1365 assert((!EnterAfter || Idx >= EnterAfter) && "Interference"); 1366 1367 selectIntv(IntvIn); 1368 Idx = leaveIntvBefore(LeaveBefore); 1369 useIntv(Start, Idx); 1370 assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); 1371 } 1372 1373 1374 void SplitEditor::splitRegInBlock(const SplitAnalysis::BlockInfo &BI, 1375 unsigned IntvIn, SlotIndex LeaveBefore) { 1376 SlotIndex Start, Stop; 1377 std::tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB); 1378 1379 DEBUG(dbgs() << "BB#" << BI.MBB->getNumber() << " [" << Start << ';' << Stop 1380 << "), uses " << BI.FirstInstr << '-' << BI.LastInstr 1381 << ", reg-in " << IntvIn << ", leave before " << LeaveBefore 1382 << (BI.LiveOut ? ", stack-out" : ", killed in block")); 1383 1384 assert(IntvIn && "Must have register in"); 1385 assert(BI.LiveIn && "Must be live-in"); 1386 assert((!LeaveBefore || LeaveBefore > Start) && "Bad interference"); 1387 1388 if (!BI.LiveOut && (!LeaveBefore || LeaveBefore >= BI.LastInstr)) { 1389 DEBUG(dbgs() << " before interference.\n"); 1390 // 1391 // <<< Interference after kill. 1392 // |---o---x | Killed in block. 1393 // ========= Use IntvIn everywhere. 1394 // 1395 selectIntv(IntvIn); 1396 useIntv(Start, BI.LastInstr); 1397 return; 1398 } 1399 1400 SlotIndex LSP = SA.getLastSplitPoint(BI.MBB->getNumber()); 1401 1402 if (!LeaveBefore || LeaveBefore > BI.LastInstr.getBoundaryIndex()) { 1403 // 1404 // <<< Possible interference after last use. 1405 // |---o---o---| Live-out on stack. 1406 // =========____ Leave IntvIn after last use. 1407 // 1408 // < Interference after last use. 1409 // |---o---o--o| Live-out on stack, late last use. 1410 // ============ Copy to stack after LSP, overlap IntvIn. 1411 // \_____ Stack interval is live-out. 1412 // 1413 if (BI.LastInstr < LSP) { 1414 DEBUG(dbgs() << ", spill after last use before interference.\n"); 1415 selectIntv(IntvIn); 1416 SlotIndex Idx = leaveIntvAfter(BI.LastInstr); 1417 useIntv(Start, Idx); 1418 assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); 1419 } else { 1420 DEBUG(dbgs() << ", spill before last split point.\n"); 1421 selectIntv(IntvIn); 1422 SlotIndex Idx = leaveIntvBefore(LSP); 1423 overlapIntv(Idx, BI.LastInstr); 1424 useIntv(Start, Idx); 1425 assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); 1426 } 1427 return; 1428 } 1429 1430 // The interference is overlapping somewhere we wanted to use IntvIn. That 1431 // means we need to create a local interval that can be allocated a 1432 // different register. 1433 unsigned LocalIntv = openIntv(); 1434 (void)LocalIntv; 1435 DEBUG(dbgs() << ", creating local interval " << LocalIntv << ".\n"); 1436 1437 if (!BI.LiveOut || BI.LastInstr < LSP) { 1438 // 1439 // <<<<<<< Interference overlapping uses. 1440 // |---o---o---| Live-out on stack. 1441 // =====----____ Leave IntvIn before interference, then spill. 1442 // 1443 SlotIndex To = leaveIntvAfter(BI.LastInstr); 1444 SlotIndex From = enterIntvBefore(LeaveBefore); 1445 useIntv(From, To); 1446 selectIntv(IntvIn); 1447 useIntv(Start, From); 1448 assert((!LeaveBefore || From <= LeaveBefore) && "Interference"); 1449 return; 1450 } 1451 1452 // <<<<<<< Interference overlapping uses. 1453 // |---o---o--o| Live-out on stack, late last use. 1454 // =====------- Copy to stack before LSP, overlap LocalIntv. 1455 // \_____ Stack interval is live-out. 1456 // 1457 SlotIndex To = leaveIntvBefore(LSP); 1458 overlapIntv(To, BI.LastInstr); 1459 SlotIndex From = enterIntvBefore(std::min(To, LeaveBefore)); 1460 useIntv(From, To); 1461 selectIntv(IntvIn); 1462 useIntv(Start, From); 1463 assert((!LeaveBefore || From <= LeaveBefore) && "Interference"); 1464 } 1465 1466 void SplitEditor::splitRegOutBlock(const SplitAnalysis::BlockInfo &BI, 1467 unsigned IntvOut, SlotIndex EnterAfter) { 1468 SlotIndex Start, Stop; 1469 std::tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB); 1470 1471 DEBUG(dbgs() << "BB#" << BI.MBB->getNumber() << " [" << Start << ';' << Stop 1472 << "), uses " << BI.FirstInstr << '-' << BI.LastInstr 1473 << ", reg-out " << IntvOut << ", enter after " << EnterAfter 1474 << (BI.LiveIn ? ", stack-in" : ", defined in block")); 1475 1476 SlotIndex LSP = SA.getLastSplitPoint(BI.MBB->getNumber()); 1477 1478 assert(IntvOut && "Must have register out"); 1479 assert(BI.LiveOut && "Must be live-out"); 1480 assert((!EnterAfter || EnterAfter < LSP) && "Bad interference"); 1481 1482 if (!BI.LiveIn && (!EnterAfter || EnterAfter <= BI.FirstInstr)) { 1483 DEBUG(dbgs() << " after interference.\n"); 1484 // 1485 // >>>> Interference before def. 1486 // | o---o---| Defined in block. 1487 // ========= Use IntvOut everywhere. 1488 // 1489 selectIntv(IntvOut); 1490 useIntv(BI.FirstInstr, Stop); 1491 return; 1492 } 1493 1494 if (!EnterAfter || EnterAfter < BI.FirstInstr.getBaseIndex()) { 1495 DEBUG(dbgs() << ", reload after interference.\n"); 1496 // 1497 // >>>> Interference before def. 1498 // |---o---o---| Live-through, stack-in. 1499 // ____========= Enter IntvOut before first use. 1500 // 1501 selectIntv(IntvOut); 1502 SlotIndex Idx = enterIntvBefore(std::min(LSP, BI.FirstInstr)); 1503 useIntv(Idx, Stop); 1504 assert((!EnterAfter || Idx >= EnterAfter) && "Interference"); 1505 return; 1506 } 1507 1508 // The interference is overlapping somewhere we wanted to use IntvOut. That 1509 // means we need to create a local interval that can be allocated a 1510 // different register. 1511 DEBUG(dbgs() << ", interference overlaps uses.\n"); 1512 // 1513 // >>>>>>> Interference overlapping uses. 1514 // |---o---o---| Live-through, stack-in. 1515 // ____---====== Create local interval for interference range. 1516 // 1517 selectIntv(IntvOut); 1518 SlotIndex Idx = enterIntvAfter(EnterAfter); 1519 useIntv(Idx, Stop); 1520 assert((!EnterAfter || Idx >= EnterAfter) && "Interference"); 1521 1522 openIntv(); 1523 SlotIndex From = enterIntvBefore(std::min(Idx, BI.FirstInstr)); 1524 useIntv(From, Idx); 1525 } 1526