1 //===- MachineCopyPropagation.cpp - Machine Copy Propagation Pass ---------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This is an extremely simple MachineInstr-level copy propagation pass. 11 // 12 // This pass forwards the source of COPYs to the users of their destinations 13 // when doing so is legal. For example: 14 // 15 // %reg1 = COPY %reg0 16 // ... 17 // ... = OP %reg1 18 // 19 // If 20 // - %reg0 has not been clobbered by the time of the use of %reg1 21 // - the register class constraints are satisfied 22 // - the COPY def is the only value that reaches OP 23 // then this pass replaces the above with: 24 // 25 // %reg1 = COPY %reg0 26 // ... 27 // ... = OP %reg0 28 // 29 // This pass also removes some redundant COPYs. For example: 30 // 31 // %R1 = COPY %R0 32 // ... // No clobber of %R1 33 // %R0 = COPY %R1 <<< Removed 34 // 35 // or 36 // 37 // %R1 = COPY %R0 38 // ... // No clobber of %R0 39 // %R1 = COPY %R0 <<< Removed 40 // 41 //===----------------------------------------------------------------------===// 42 43 #include "llvm/ADT/DenseMap.h" 44 #include "llvm/ADT/STLExtras.h" 45 #include "llvm/ADT/SetVector.h" 46 #include "llvm/ADT/SmallVector.h" 47 #include "llvm/ADT/Statistic.h" 48 #include "llvm/ADT/iterator_range.h" 49 #include "llvm/CodeGen/MachineBasicBlock.h" 50 #include "llvm/CodeGen/MachineFunction.h" 51 #include "llvm/CodeGen/MachineFunctionPass.h" 52 #include "llvm/CodeGen/MachineInstr.h" 53 #include "llvm/CodeGen/MachineOperand.h" 54 #include "llvm/CodeGen/MachineRegisterInfo.h" 55 #include "llvm/CodeGen/TargetInstrInfo.h" 56 #include "llvm/CodeGen/TargetRegisterInfo.h" 57 #include "llvm/CodeGen/TargetSubtargetInfo.h" 58 #include "llvm/MC/MCRegisterInfo.h" 59 #include "llvm/Pass.h" 60 #include "llvm/Support/Debug.h" 61 #include "llvm/Support/DebugCounter.h" 62 #include "llvm/Support/raw_ostream.h" 63 #include <cassert> 64 #include <iterator> 65 66 using namespace llvm; 67 68 #define DEBUG_TYPE "machine-cp" 69 70 STATISTIC(NumDeletes, "Number of dead copies deleted"); 71 STATISTIC(NumCopyForwards, "Number of copy uses forwarded"); 72 DEBUG_COUNTER(FwdCounter, "machine-cp-fwd", 73 "Controls which register COPYs are forwarded"); 74 75 namespace { 76 77 using RegList = SmallVector<unsigned, 4>; 78 using SourceMap = DenseMap<unsigned, RegList>; 79 using Reg2MIMap = DenseMap<unsigned, MachineInstr *>; 80 81 class MachineCopyPropagation : public MachineFunctionPass { 82 const TargetRegisterInfo *TRI; 83 const TargetInstrInfo *TII; 84 const MachineRegisterInfo *MRI; 85 86 public: 87 static char ID; // Pass identification, replacement for typeid 88 89 MachineCopyPropagation() : MachineFunctionPass(ID) { 90 initializeMachineCopyPropagationPass(*PassRegistry::getPassRegistry()); 91 } 92 93 void getAnalysisUsage(AnalysisUsage &AU) const override { 94 AU.setPreservesCFG(); 95 MachineFunctionPass::getAnalysisUsage(AU); 96 } 97 98 bool runOnMachineFunction(MachineFunction &MF) override; 99 100 MachineFunctionProperties getRequiredProperties() const override { 101 return MachineFunctionProperties().set( 102 MachineFunctionProperties::Property::NoVRegs); 103 } 104 105 private: 106 void ClobberRegister(unsigned Reg); 107 void ReadRegister(unsigned Reg); 108 void CopyPropagateBlock(MachineBasicBlock &MBB); 109 bool eraseIfRedundant(MachineInstr &Copy, unsigned Src, unsigned Def); 110 void forwardUses(MachineInstr &MI); 111 bool isForwardableRegClassCopy(const MachineInstr &Copy, 112 const MachineInstr &UseI, unsigned UseIdx); 113 bool hasImplicitOverlap(const MachineInstr &MI, const MachineOperand &Use); 114 115 /// Candidates for deletion. 116 SmallSetVector<MachineInstr*, 8> MaybeDeadCopies; 117 118 /// Def -> available copies map. 119 Reg2MIMap AvailCopyMap; 120 121 /// Def -> copies map. 122 Reg2MIMap CopyMap; 123 124 /// Src -> Def map 125 SourceMap SrcMap; 126 127 bool Changed; 128 }; 129 130 } // end anonymous namespace 131 132 char MachineCopyPropagation::ID = 0; 133 134 char &llvm::MachineCopyPropagationID = MachineCopyPropagation::ID; 135 136 INITIALIZE_PASS(MachineCopyPropagation, DEBUG_TYPE, 137 "Machine Copy Propagation Pass", false, false) 138 139 /// Remove any entry in \p Map where the register is a subregister or equal to 140 /// a register contained in \p Regs. 141 static void removeRegsFromMap(Reg2MIMap &Map, const RegList &Regs, 142 const TargetRegisterInfo &TRI) { 143 for (unsigned Reg : Regs) { 144 // Source of copy is no longer available for propagation. 145 for (MCSubRegIterator SR(Reg, &TRI, true); SR.isValid(); ++SR) 146 Map.erase(*SR); 147 } 148 } 149 150 /// Remove any entry in \p Map that is marked clobbered in \p RegMask. 151 /// The map will typically have a lot fewer entries than the regmask clobbers, 152 /// so this is more efficient than iterating the clobbered registers and calling 153 /// ClobberRegister() on them. 154 static void removeClobberedRegsFromMap(Reg2MIMap &Map, 155 const MachineOperand &RegMask) { 156 for (Reg2MIMap::iterator I = Map.begin(), E = Map.end(), Next; I != E; 157 I = Next) { 158 Next = std::next(I); 159 unsigned Reg = I->first; 160 if (RegMask.clobbersPhysReg(Reg)) 161 Map.erase(I); 162 } 163 } 164 165 void MachineCopyPropagation::ClobberRegister(unsigned Reg) { 166 for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) { 167 CopyMap.erase(*AI); 168 AvailCopyMap.erase(*AI); 169 170 SourceMap::iterator SI = SrcMap.find(*AI); 171 if (SI != SrcMap.end()) { 172 removeRegsFromMap(AvailCopyMap, SI->second, *TRI); 173 SrcMap.erase(SI); 174 } 175 } 176 } 177 178 void MachineCopyPropagation::ReadRegister(unsigned Reg) { 179 // If 'Reg' is defined by a copy, the copy is no longer a candidate 180 // for elimination. 181 for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) { 182 Reg2MIMap::iterator CI = CopyMap.find(*AI); 183 if (CI != CopyMap.end()) { 184 LLVM_DEBUG(dbgs() << "MCP: Copy is used - not dead: "; 185 CI->second->dump()); 186 MaybeDeadCopies.remove(CI->second); 187 } 188 } 189 } 190 191 /// Return true if \p PreviousCopy did copy register \p Src to register \p Def. 192 /// This fact may have been obscured by sub register usage or may not be true at 193 /// all even though Src and Def are subregisters of the registers used in 194 /// PreviousCopy. e.g. 195 /// isNopCopy("ecx = COPY eax", AX, CX) == true 196 /// isNopCopy("ecx = COPY eax", AH, CL) == false 197 static bool isNopCopy(const MachineInstr &PreviousCopy, unsigned Src, 198 unsigned Def, const TargetRegisterInfo *TRI) { 199 unsigned PreviousSrc = PreviousCopy.getOperand(1).getReg(); 200 unsigned PreviousDef = PreviousCopy.getOperand(0).getReg(); 201 if (Src == PreviousSrc) { 202 assert(Def == PreviousDef); 203 return true; 204 } 205 if (!TRI->isSubRegister(PreviousSrc, Src)) 206 return false; 207 unsigned SubIdx = TRI->getSubRegIndex(PreviousSrc, Src); 208 return SubIdx == TRI->getSubRegIndex(PreviousDef, Def); 209 } 210 211 /// Remove instruction \p Copy if there exists a previous copy that copies the 212 /// register \p Src to the register \p Def; This may happen indirectly by 213 /// copying the super registers. 214 bool MachineCopyPropagation::eraseIfRedundant(MachineInstr &Copy, unsigned Src, 215 unsigned Def) { 216 // Avoid eliminating a copy from/to a reserved registers as we cannot predict 217 // the value (Example: The sparc zero register is writable but stays zero). 218 if (MRI->isReserved(Src) || MRI->isReserved(Def)) 219 return false; 220 221 // Search for an existing copy. 222 Reg2MIMap::iterator CI = AvailCopyMap.find(Def); 223 if (CI == AvailCopyMap.end()) 224 return false; 225 226 // Check that the existing copy uses the correct sub registers. 227 MachineInstr &PrevCopy = *CI->second; 228 if (PrevCopy.getOperand(0).isDead()) 229 return false; 230 if (!isNopCopy(PrevCopy, Src, Def, TRI)) 231 return false; 232 233 LLVM_DEBUG(dbgs() << "MCP: copy is a NOP, removing: "; Copy.dump()); 234 235 // Copy was redundantly redefining either Src or Def. Remove earlier kill 236 // flags between Copy and PrevCopy because the value will be reused now. 237 assert(Copy.isCopy()); 238 unsigned CopyDef = Copy.getOperand(0).getReg(); 239 assert(CopyDef == Src || CopyDef == Def); 240 for (MachineInstr &MI : 241 make_range(PrevCopy.getIterator(), Copy.getIterator())) 242 MI.clearRegisterKills(CopyDef, TRI); 243 244 Copy.eraseFromParent(); 245 Changed = true; 246 ++NumDeletes; 247 return true; 248 } 249 250 /// Decide whether we should forward the source of \param Copy to its use in 251 /// \param UseI based on the physical register class constraints of the opcode 252 /// and avoiding introducing more cross-class COPYs. 253 bool MachineCopyPropagation::isForwardableRegClassCopy(const MachineInstr &Copy, 254 const MachineInstr &UseI, 255 unsigned UseIdx) { 256 257 unsigned CopySrcReg = Copy.getOperand(1).getReg(); 258 259 // If the new register meets the opcode register constraints, then allow 260 // forwarding. 261 if (const TargetRegisterClass *URC = 262 UseI.getRegClassConstraint(UseIdx, TII, TRI)) 263 return URC->contains(CopySrcReg); 264 265 if (!UseI.isCopy()) 266 return false; 267 268 /// COPYs don't have register class constraints, so if the user instruction 269 /// is a COPY, we just try to avoid introducing additional cross-class 270 /// COPYs. For example: 271 /// 272 /// RegClassA = COPY RegClassB // Copy parameter 273 /// ... 274 /// RegClassB = COPY RegClassA // UseI parameter 275 /// 276 /// which after forwarding becomes 277 /// 278 /// RegClassA = COPY RegClassB 279 /// ... 280 /// RegClassB = COPY RegClassB 281 /// 282 /// so we have reduced the number of cross-class COPYs and potentially 283 /// introduced a nop COPY that can be removed. 284 const TargetRegisterClass *UseDstRC = 285 TRI->getMinimalPhysRegClass(UseI.getOperand(0).getReg()); 286 287 const TargetRegisterClass *SuperRC = UseDstRC; 288 for (TargetRegisterClass::sc_iterator SuperRCI = UseDstRC->getSuperClasses(); 289 SuperRC; SuperRC = *SuperRCI++) 290 if (SuperRC->contains(CopySrcReg)) 291 return true; 292 293 return false; 294 } 295 296 /// Check that \p MI does not have implicit uses that overlap with it's \p Use 297 /// operand (the register being replaced), since these can sometimes be 298 /// implicitly tied to other operands. For example, on AMDGPU: 299 /// 300 /// V_MOVRELS_B32_e32 %VGPR2, %M0<imp-use>, %EXEC<imp-use>, %VGPR2_VGPR3_VGPR4_VGPR5<imp-use> 301 /// 302 /// the %VGPR2 is implicitly tied to the larger reg operand, but we have no 303 /// way of knowing we need to update the latter when updating the former. 304 bool MachineCopyPropagation::hasImplicitOverlap(const MachineInstr &MI, 305 const MachineOperand &Use) { 306 for (const MachineOperand &MIUse : MI.uses()) 307 if (&MIUse != &Use && MIUse.isReg() && MIUse.isImplicit() && 308 MIUse.isUse() && TRI->regsOverlap(Use.getReg(), MIUse.getReg())) 309 return true; 310 311 return false; 312 } 313 314 /// Look for available copies whose destination register is used by \p MI and 315 /// replace the use in \p MI with the copy's source register. 316 void MachineCopyPropagation::forwardUses(MachineInstr &MI) { 317 if (AvailCopyMap.empty()) 318 return; 319 320 // Look for non-tied explicit vreg uses that have an active COPY 321 // instruction that defines the physical register allocated to them. 322 // Replace the vreg with the source of the active COPY. 323 for (unsigned OpIdx = 0, OpEnd = MI.getNumOperands(); OpIdx < OpEnd; 324 ++OpIdx) { 325 MachineOperand &MOUse = MI.getOperand(OpIdx); 326 // Don't forward into undef use operands since doing so can cause problems 327 // with the machine verifier, since it doesn't treat undef reads as reads, 328 // so we can end up with a live range that ends on an undef read, leading to 329 // an error that the live range doesn't end on a read of the live range 330 // register. 331 if (!MOUse.isReg() || MOUse.isTied() || MOUse.isUndef() || MOUse.isDef() || 332 MOUse.isImplicit()) 333 continue; 334 335 if (!MOUse.getReg()) 336 continue; 337 338 // Check that the register is marked 'renamable' so we know it is safe to 339 // rename it without violating any constraints that aren't expressed in the 340 // IR (e.g. ABI or opcode requirements). 341 if (!MOUse.isRenamable()) 342 continue; 343 344 auto CI = AvailCopyMap.find(MOUse.getReg()); 345 if (CI == AvailCopyMap.end()) 346 continue; 347 348 MachineInstr &Copy = *CI->second; 349 unsigned CopyDstReg = Copy.getOperand(0).getReg(); 350 const MachineOperand &CopySrc = Copy.getOperand(1); 351 unsigned CopySrcReg = CopySrc.getReg(); 352 353 // FIXME: Don't handle partial uses of wider COPYs yet. 354 if (MOUse.getReg() != CopyDstReg) { 355 LLVM_DEBUG( 356 dbgs() << "MCP: FIXME! Not forwarding COPY to sub-register use:\n " 357 << MI); 358 continue; 359 } 360 361 // Don't forward COPYs of reserved regs unless they are constant. 362 if (MRI->isReserved(CopySrcReg) && !MRI->isConstantPhysReg(CopySrcReg)) 363 continue; 364 365 if (!isForwardableRegClassCopy(Copy, MI, OpIdx)) 366 continue; 367 368 if (hasImplicitOverlap(MI, MOUse)) 369 continue; 370 371 if (!DebugCounter::shouldExecute(FwdCounter)) { 372 LLVM_DEBUG(dbgs() << "MCP: Skipping forwarding due to debug counter:\n " 373 << MI); 374 continue; 375 } 376 377 LLVM_DEBUG(dbgs() << "MCP: Replacing " << printReg(MOUse.getReg(), TRI) 378 << "\n with " << printReg(CopySrcReg, TRI) 379 << "\n in " << MI << " from " << Copy); 380 381 MOUse.setReg(CopySrcReg); 382 if (!CopySrc.isRenamable()) 383 MOUse.setIsRenamable(false); 384 385 LLVM_DEBUG(dbgs() << "MCP: After replacement: " << MI << "\n"); 386 387 // Clear kill markers that may have been invalidated. 388 for (MachineInstr &KMI : 389 make_range(Copy.getIterator(), std::next(MI.getIterator()))) 390 KMI.clearRegisterKills(CopySrcReg, TRI); 391 392 ++NumCopyForwards; 393 Changed = true; 394 } 395 } 396 397 void MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) { 398 LLVM_DEBUG(dbgs() << "MCP: CopyPropagateBlock " << MBB.getName() << "\n"); 399 400 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E; ) { 401 MachineInstr *MI = &*I; 402 ++I; 403 404 // Analyze copies (which don't overlap themselves). 405 if (MI->isCopy() && !TRI->regsOverlap(MI->getOperand(0).getReg(), 406 MI->getOperand(1).getReg())) { 407 unsigned Def = MI->getOperand(0).getReg(); 408 unsigned Src = MI->getOperand(1).getReg(); 409 410 assert(!TargetRegisterInfo::isVirtualRegister(Def) && 411 !TargetRegisterInfo::isVirtualRegister(Src) && 412 "MachineCopyPropagation should be run after register allocation!"); 413 414 // The two copies cancel out and the source of the first copy 415 // hasn't been overridden, eliminate the second one. e.g. 416 // %ecx = COPY %eax 417 // ... nothing clobbered eax. 418 // %eax = COPY %ecx 419 // => 420 // %ecx = COPY %eax 421 // 422 // or 423 // 424 // %ecx = COPY %eax 425 // ... nothing clobbered eax. 426 // %ecx = COPY %eax 427 // => 428 // %ecx = COPY %eax 429 if (eraseIfRedundant(*MI, Def, Src) || eraseIfRedundant(*MI, Src, Def)) 430 continue; 431 432 forwardUses(*MI); 433 434 // Src may have been changed by forwardUses() 435 Src = MI->getOperand(1).getReg(); 436 437 // If Src is defined by a previous copy, the previous copy cannot be 438 // eliminated. 439 ReadRegister(Src); 440 for (const MachineOperand &MO : MI->implicit_operands()) { 441 if (!MO.isReg() || !MO.readsReg()) 442 continue; 443 unsigned Reg = MO.getReg(); 444 if (!Reg) 445 continue; 446 ReadRegister(Reg); 447 } 448 449 LLVM_DEBUG(dbgs() << "MCP: Copy is a deletion candidate: "; MI->dump()); 450 451 // Copy is now a candidate for deletion. 452 if (!MRI->isReserved(Def)) 453 MaybeDeadCopies.insert(MI); 454 455 // If 'Def' is previously source of another copy, then this earlier copy's 456 // source is no longer available. e.g. 457 // %xmm9 = copy %xmm2 458 // ... 459 // %xmm2 = copy %xmm0 460 // ... 461 // %xmm2 = copy %xmm9 462 ClobberRegister(Def); 463 for (const MachineOperand &MO : MI->implicit_operands()) { 464 if (!MO.isReg() || !MO.isDef()) 465 continue; 466 unsigned Reg = MO.getReg(); 467 if (!Reg) 468 continue; 469 ClobberRegister(Reg); 470 } 471 472 // Remember Def is defined by the copy. 473 for (MCSubRegIterator SR(Def, TRI, /*IncludeSelf=*/true); SR.isValid(); 474 ++SR) { 475 CopyMap[*SR] = MI; 476 AvailCopyMap[*SR] = MI; 477 } 478 479 // Remember source that's copied to Def. Once it's clobbered, then 480 // it's no longer available for copy propagation. 481 RegList &DestList = SrcMap[Src]; 482 if (!is_contained(DestList, Def)) 483 DestList.push_back(Def); 484 485 continue; 486 } 487 488 // Clobber any earlyclobber regs first. 489 for (const MachineOperand &MO : MI->operands()) 490 if (MO.isReg() && MO.isEarlyClobber()) { 491 unsigned Reg = MO.getReg(); 492 // If we have a tied earlyclobber, that means it is also read by this 493 // instruction, so we need to make sure we don't remove it as dead 494 // later. 495 if (MO.isTied()) 496 ReadRegister(Reg); 497 ClobberRegister(Reg); 498 } 499 500 forwardUses(*MI); 501 502 // Not a copy. 503 SmallVector<unsigned, 2> Defs; 504 const MachineOperand *RegMask = nullptr; 505 for (const MachineOperand &MO : MI->operands()) { 506 if (MO.isRegMask()) 507 RegMask = &MO; 508 if (!MO.isReg()) 509 continue; 510 unsigned Reg = MO.getReg(); 511 if (!Reg) 512 continue; 513 514 assert(!TargetRegisterInfo::isVirtualRegister(Reg) && 515 "MachineCopyPropagation should be run after register allocation!"); 516 517 if (MO.isDef() && !MO.isEarlyClobber()) { 518 Defs.push_back(Reg); 519 continue; 520 } else if (!MO.isDebug() && MO.readsReg()) 521 ReadRegister(Reg); 522 } 523 524 // The instruction has a register mask operand which means that it clobbers 525 // a large set of registers. Treat clobbered registers the same way as 526 // defined registers. 527 if (RegMask) { 528 // Erase any MaybeDeadCopies whose destination register is clobbered. 529 for (SmallSetVector<MachineInstr *, 8>::iterator DI = 530 MaybeDeadCopies.begin(); 531 DI != MaybeDeadCopies.end();) { 532 MachineInstr *MaybeDead = *DI; 533 unsigned Reg = MaybeDead->getOperand(0).getReg(); 534 assert(!MRI->isReserved(Reg)); 535 536 if (!RegMask->clobbersPhysReg(Reg)) { 537 ++DI; 538 continue; 539 } 540 541 LLVM_DEBUG(dbgs() << "MCP: Removing copy due to regmask clobbering: "; 542 MaybeDead->dump()); 543 544 // erase() will return the next valid iterator pointing to the next 545 // element after the erased one. 546 DI = MaybeDeadCopies.erase(DI); 547 MaybeDead->eraseFromParent(); 548 Changed = true; 549 ++NumDeletes; 550 } 551 552 removeClobberedRegsFromMap(AvailCopyMap, *RegMask); 553 removeClobberedRegsFromMap(CopyMap, *RegMask); 554 for (SourceMap::iterator I = SrcMap.begin(), E = SrcMap.end(), Next; 555 I != E; I = Next) { 556 Next = std::next(I); 557 if (RegMask->clobbersPhysReg(I->first)) { 558 removeRegsFromMap(AvailCopyMap, I->second, *TRI); 559 SrcMap.erase(I); 560 } 561 } 562 } 563 564 // Any previous copy definition or reading the Defs is no longer available. 565 for (unsigned Reg : Defs) 566 ClobberRegister(Reg); 567 } 568 569 // If MBB doesn't have successors, delete the copies whose defs are not used. 570 // If MBB does have successors, then conservative assume the defs are live-out 571 // since we don't want to trust live-in lists. 572 if (MBB.succ_empty()) { 573 for (MachineInstr *MaybeDead : MaybeDeadCopies) { 574 LLVM_DEBUG(dbgs() << "MCP: Removing copy due to no live-out succ: "; 575 MaybeDead->dump()); 576 assert(!MRI->isReserved(MaybeDead->getOperand(0).getReg())); 577 MaybeDead->eraseFromParent(); 578 Changed = true; 579 ++NumDeletes; 580 } 581 } 582 583 MaybeDeadCopies.clear(); 584 AvailCopyMap.clear(); 585 CopyMap.clear(); 586 SrcMap.clear(); 587 } 588 589 bool MachineCopyPropagation::runOnMachineFunction(MachineFunction &MF) { 590 if (skipFunction(MF.getFunction())) 591 return false; 592 593 Changed = false; 594 595 TRI = MF.getSubtarget().getRegisterInfo(); 596 TII = MF.getSubtarget().getInstrInfo(); 597 MRI = &MF.getRegInfo(); 598 599 for (MachineBasicBlock &MBB : MF) 600 CopyPropagateBlock(MBB); 601 602 return Changed; 603 } 604