1 //===-- AArch64A57FPLoadBalancing.cpp - Balance FP ops statically on A57---===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // For best-case performance on Cortex-A57, we should try to use a balanced 10 // mix of odd and even D-registers when performing a critical sequence of 11 // independent, non-quadword FP/ASIMD floating-point multiply or 12 // multiply-accumulate operations. 13 // 14 // This pass attempts to detect situations where the register allocation may 15 // adversely affect this load balancing and to change the registers used so as 16 // to better utilize the CPU. 17 // 18 // Ideally we'd just take each multiply or multiply-accumulate in turn and 19 // allocate it alternating even or odd registers. However, multiply-accumulates 20 // are most efficiently performed in the same functional unit as their 21 // accumulation operand. Therefore this pass tries to find maximal sequences 22 // ("Chains") of multiply-accumulates linked via their accumulation operand, 23 // and assign them all the same "color" (oddness/evenness). 24 // 25 // This optimization affects S-register and D-register floating point 26 // multiplies and FMADD/FMAs, as well as vector (floating point only) muls and 27 // FMADD/FMA. Q register instructions (and 128-bit vector instructions) are 28 // not affected. 29 //===----------------------------------------------------------------------===// 30 31 #include "AArch64.h" 32 #include "AArch64InstrInfo.h" 33 #include "AArch64Subtarget.h" 34 #include "llvm/ADT/BitVector.h" 35 #include "llvm/ADT/EquivalenceClasses.h" 36 #include "llvm/CodeGen/MachineFunction.h" 37 #include "llvm/CodeGen/MachineFunctionPass.h" 38 #include "llvm/CodeGen/MachineInstr.h" 39 #include "llvm/CodeGen/MachineInstrBuilder.h" 40 #include "llvm/CodeGen/MachineRegisterInfo.h" 41 #include "llvm/CodeGen/RegisterScavenging.h" 42 #include "llvm/CodeGen/RegisterClassInfo.h" 43 #include "llvm/Support/CommandLine.h" 44 #include "llvm/Support/Debug.h" 45 #include "llvm/Support/raw_ostream.h" 46 #include <list> 47 using namespace llvm; 48 49 #define DEBUG_TYPE "aarch64-a57-fp-load-balancing" 50 51 // Enforce the algorithm to use the scavenged register even when the original 52 // destination register is the correct color. Used for testing. 53 static cl::opt<bool> 54 TransformAll("aarch64-a57-fp-load-balancing-force-all", 55 cl::desc("Always modify dest registers regardless of color"), 56 cl::init(false), cl::Hidden); 57 58 // Never use the balance information obtained from chains - return a specific 59 // color always. Used for testing. 60 static cl::opt<unsigned> 61 OverrideBalance("aarch64-a57-fp-load-balancing-override", 62 cl::desc("Ignore balance information, always return " 63 "(1: Even, 2: Odd)."), 64 cl::init(0), cl::Hidden); 65 66 //===----------------------------------------------------------------------===// 67 // Helper functions 68 69 // Is the instruction a type of multiply on 64-bit (or 32-bit) FPRs? 70 static bool isMul(MachineInstr *MI) { 71 switch (MI->getOpcode()) { 72 case AArch64::FMULSrr: 73 case AArch64::FNMULSrr: 74 case AArch64::FMULDrr: 75 case AArch64::FNMULDrr: 76 return true; 77 default: 78 return false; 79 } 80 } 81 82 // Is the instruction a type of FP multiply-accumulate on 64-bit (or 32-bit) FPRs? 83 static bool isMla(MachineInstr *MI) { 84 switch (MI->getOpcode()) { 85 case AArch64::FMSUBSrrr: 86 case AArch64::FMADDSrrr: 87 case AArch64::FNMSUBSrrr: 88 case AArch64::FNMADDSrrr: 89 case AArch64::FMSUBDrrr: 90 case AArch64::FMADDDrrr: 91 case AArch64::FNMSUBDrrr: 92 case AArch64::FNMADDDrrr: 93 return true; 94 default: 95 return false; 96 } 97 } 98 99 //===----------------------------------------------------------------------===// 100 101 namespace { 102 /// A "color", which is either even or odd. Yes, these aren't really colors 103 /// but the algorithm is conceptually doing two-color graph coloring. 104 enum class Color { Even, Odd }; 105 static const char *ColorNames[2] = { "Even", "Odd" }; 106 107 class Chain; 108 109 class AArch64A57FPLoadBalancing : public MachineFunctionPass { 110 const AArch64InstrInfo *TII; 111 MachineRegisterInfo *MRI; 112 const TargetRegisterInfo *TRI; 113 RegisterClassInfo RCI; 114 115 public: 116 static char ID; 117 explicit AArch64A57FPLoadBalancing() : MachineFunctionPass(ID) {} 118 119 bool runOnMachineFunction(MachineFunction &F) override; 120 121 const char *getPassName() const override { 122 return "A57 FP Anti-dependency breaker"; 123 } 124 125 void getAnalysisUsage(AnalysisUsage &AU) const override { 126 AU.setPreservesCFG(); 127 MachineFunctionPass::getAnalysisUsage(AU); 128 } 129 130 private: 131 bool runOnBasicBlock(MachineBasicBlock &MBB); 132 bool colorChainSet(std::vector<Chain*> GV, MachineBasicBlock &MBB, 133 int &Balance); 134 bool colorChain(Chain *G, Color C, MachineBasicBlock &MBB); 135 int scavengeRegister(Chain *G, Color C, MachineBasicBlock &MBB); 136 void scanInstruction(MachineInstr *MI, unsigned Idx, 137 std::map<unsigned, Chain*> &Active, 138 std::set<std::unique_ptr<Chain>> &AllChains); 139 void maybeKillChain(MachineOperand &MO, unsigned Idx, 140 std::map<unsigned, Chain*> &RegChains); 141 Color getColor(unsigned Register); 142 Chain *getAndEraseNext(Color PreferredColor, std::vector<Chain*> &L); 143 }; 144 char AArch64A57FPLoadBalancing::ID = 0; 145 146 /// A Chain is a sequence of instructions that are linked together by 147 /// an accumulation operand. For example: 148 /// 149 /// fmul d0<def>, ? 150 /// fmla d1<def>, ?, ?, d0<kill> 151 /// fmla d2<def>, ?, ?, d1<kill> 152 /// 153 /// There may be other instructions interleaved in the sequence that 154 /// do not belong to the chain. These other instructions must not use 155 /// the "chain" register at any point. 156 /// 157 /// We currently only support chains where the "chain" operand is killed 158 /// at each link in the chain for simplicity. 159 /// A chain has three important instructions - Start, Last and Kill. 160 /// * The start instruction is the first instruction in the chain. 161 /// * Last is the final instruction in the chain. 162 /// * Kill may or may not be defined. If defined, Kill is the instruction 163 /// where the outgoing value of the Last instruction is killed. 164 /// This information is important as if we know the outgoing value is 165 /// killed with no intervening uses, we can safely change its register. 166 /// 167 /// Without a kill instruction, we must assume the outgoing value escapes 168 /// beyond our model and either must not change its register or must 169 /// create a fixup FMOV to keep the old register value consistent. 170 /// 171 class Chain { 172 public: 173 /// The important (marker) instructions. 174 MachineInstr *StartInst, *LastInst, *KillInst; 175 /// The index, from the start of the basic block, that each marker 176 /// appears. These are stored so we can do quick interval tests. 177 unsigned StartInstIdx, LastInstIdx, KillInstIdx; 178 /// All instructions in the chain. 179 std::set<MachineInstr*> Insts; 180 /// True if KillInst cannot be modified. If this is true, 181 /// we cannot change LastInst's outgoing register. 182 /// This will be true for tied values and regmasks. 183 bool KillIsImmutable; 184 /// The "color" of LastInst. This will be the preferred chain color, 185 /// as changing intermediate nodes is easy but changing the last 186 /// instruction can be more tricky. 187 Color LastColor; 188 189 Chain(MachineInstr *MI, unsigned Idx, Color C) 190 : StartInst(MI), LastInst(MI), KillInst(nullptr), 191 StartInstIdx(Idx), LastInstIdx(Idx), KillInstIdx(0), 192 LastColor(C) { 193 Insts.insert(MI); 194 } 195 196 /// Add a new instruction into the chain. The instruction's dest operand 197 /// has the given color. 198 void add(MachineInstr *MI, unsigned Idx, Color C) { 199 LastInst = MI; 200 LastInstIdx = Idx; 201 LastColor = C; 202 assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) && 203 "Chain: broken invariant. A Chain can only be killed after its last " 204 "def"); 205 206 Insts.insert(MI); 207 } 208 209 /// Return true if MI is a member of the chain. 210 bool contains(MachineInstr *MI) { return Insts.count(MI) > 0; } 211 212 /// Return the number of instructions in the chain. 213 unsigned size() const { 214 return Insts.size(); 215 } 216 217 /// Inform the chain that its last active register (the dest register of 218 /// LastInst) is killed by MI with no intervening uses or defs. 219 void setKill(MachineInstr *MI, unsigned Idx, bool Immutable) { 220 KillInst = MI; 221 KillInstIdx = Idx; 222 KillIsImmutable = Immutable; 223 assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) && 224 "Chain: broken invariant. A Chain can only be killed after its last " 225 "def"); 226 } 227 228 /// Return the first instruction in the chain. 229 MachineInstr *getStart() const { return StartInst; } 230 /// Return the last instruction in the chain. 231 MachineInstr *getLast() const { return LastInst; } 232 /// Return the "kill" instruction (as set with setKill()) or NULL. 233 MachineInstr *getKill() const { return KillInst; } 234 /// Return an instruction that can be used as an iterator for the end 235 /// of the chain. This is the maximum of KillInst (if set) and LastInst. 236 MachineBasicBlock::iterator getEnd() const { 237 return ++MachineBasicBlock::iterator(KillInst ? KillInst : LastInst); 238 } 239 240 /// Can the Kill instruction (assuming one exists) be modified? 241 bool isKillImmutable() const { return KillIsImmutable; } 242 243 /// Return the preferred color of this chain. 244 Color getPreferredColor() { 245 if (OverrideBalance != 0) 246 return OverrideBalance == 1 ? Color::Even : Color::Odd; 247 return LastColor; 248 } 249 250 /// Return true if this chain (StartInst..KillInst) overlaps with Other. 251 bool rangeOverlapsWith(const Chain &Other) const { 252 unsigned End = KillInst ? KillInstIdx : LastInstIdx; 253 unsigned OtherEnd = Other.KillInst ? 254 Other.KillInstIdx : Other.LastInstIdx; 255 256 return StartInstIdx <= OtherEnd && Other.StartInstIdx <= End; 257 } 258 259 /// Return true if this chain starts before Other. 260 bool startsBefore(Chain *Other) { 261 return StartInstIdx < Other->StartInstIdx; 262 } 263 264 /// Return true if the group will require a fixup MOV at the end. 265 bool requiresFixup() const { 266 return (getKill() && isKillImmutable()) || !getKill(); 267 } 268 269 /// Return a simple string representation of the chain. 270 std::string str() const { 271 std::string S; 272 raw_string_ostream OS(S); 273 274 OS << "{"; 275 StartInst->print(OS, NULL, true); 276 OS << " -> "; 277 LastInst->print(OS, NULL, true); 278 if (KillInst) { 279 OS << " (kill @ "; 280 KillInst->print(OS, NULL, true); 281 OS << ")"; 282 } 283 OS << "}"; 284 285 return OS.str(); 286 } 287 288 }; 289 290 } // end anonymous namespace 291 292 //===----------------------------------------------------------------------===// 293 294 bool AArch64A57FPLoadBalancing::runOnMachineFunction(MachineFunction &F) { 295 bool Changed = false; 296 DEBUG(dbgs() << "***** AArch64A57FPLoadBalancing *****\n"); 297 298 const TargetMachine &TM = F.getTarget(); 299 MRI = &F.getRegInfo(); 300 TRI = F.getRegInfo().getTargetRegisterInfo(); 301 TII = TM.getSubtarget<AArch64Subtarget>().getInstrInfo(); 302 RCI.runOnMachineFunction(F); 303 304 for (auto &MBB : F) { 305 Changed |= runOnBasicBlock(MBB); 306 } 307 308 return Changed; 309 } 310 311 bool AArch64A57FPLoadBalancing::runOnBasicBlock(MachineBasicBlock &MBB) { 312 bool Changed = false; 313 DEBUG(dbgs() << "Running on MBB: " << MBB << " - scanning instructions...\n"); 314 315 // First, scan the basic block producing a set of chains. 316 317 // The currently "active" chains - chains that can be added to and haven't 318 // been killed yet. This is keyed by register - all chains can only have one 319 // "link" register between each inst in the chain. 320 std::map<unsigned, Chain*> ActiveChains; 321 std::set<std::unique_ptr<Chain>> AllChains; 322 unsigned Idx = 0; 323 for (auto &MI : MBB) 324 scanInstruction(&MI, Idx++, ActiveChains, AllChains); 325 326 DEBUG(dbgs() << "Scan complete, "<< AllChains.size() << " chains created.\n"); 327 328 // Group the chains into disjoint sets based on their liveness range. This is 329 // a poor-man's version of graph coloring. Ideally we'd create an interference 330 // graph and perform full-on graph coloring on that, but; 331 // (a) That's rather heavyweight for only two colors. 332 // (b) We expect multiple disjoint interference regions - in practice the live 333 // range of chains is quite small and they are clustered between loads 334 // and stores. 335 EquivalenceClasses<Chain*> EC; 336 for (auto &I : AllChains) 337 EC.insert(I.get()); 338 339 for (auto &I : AllChains) 340 for (auto &J : AllChains) 341 if (I != J && I->rangeOverlapsWith(*J)) 342 EC.unionSets(I.get(), J.get()); 343 DEBUG(dbgs() << "Created " << EC.getNumClasses() << " disjoint sets.\n"); 344 345 // Now we assume that every member of an equivalence class interferes 346 // with every other member of that class, and with no members of other classes. 347 348 // Convert the EquivalenceClasses to a simpler set of sets. 349 std::vector<std::vector<Chain*> > V; 350 for (auto I = EC.begin(), E = EC.end(); I != E; ++I) { 351 std::vector<Chain*> Cs(EC.member_begin(I), EC.member_end()); 352 if (Cs.empty()) continue; 353 V.push_back(Cs); 354 } 355 356 // Now we have a set of sets, order them by start address so 357 // we can iterate over them sequentially. 358 std::sort(V.begin(), V.end(), 359 [](const std::vector<Chain*> &A, 360 const std::vector<Chain*> &B) { 361 return A.front()->startsBefore(B.front()); 362 }); 363 364 // As we only have two colors, we can track the global (BB-level) balance of 365 // odds versus evens. We aim to keep this near zero to keep both execution 366 // units fed. 367 // Positive means we're even-heavy, negative we're odd-heavy. 368 // 369 // FIXME: If chains have interdependencies, for example: 370 // mul r0, r1, r2 371 // mul r3, r0, r1 372 // We do not model this and may color each one differently, assuming we'll 373 // get ILP when we obviously can't. This hasn't been seen to be a problem 374 // in practice so far, so we simplify the algorithm by ignoring it. 375 int Parity = 0; 376 377 for (auto &I : V) 378 Changed |= colorChainSet(I, MBB, Parity); 379 380 return Changed; 381 } 382 383 Chain *AArch64A57FPLoadBalancing::getAndEraseNext(Color PreferredColor, 384 std::vector<Chain*> &L) { 385 if (L.empty()) 386 return nullptr; 387 388 // We try and get the best candidate from L to color next, given that our 389 // preferred color is "PreferredColor". L is ordered from larger to smaller 390 // chains. It is beneficial to color the large chains before the small chains, 391 // but if we can't find a chain of the maximum length with the preferred color, 392 // we fuzz the size and look for slightly smaller chains before giving up and 393 // returning a chain that must be recolored. 394 395 // FIXME: Does this need to be configurable? 396 const unsigned SizeFuzz = 1; 397 unsigned MinSize = L.front()->size() - SizeFuzz; 398 for (auto I = L.begin(), E = L.end(); I != E; ++I) { 399 if ((*I)->size() <= MinSize) { 400 // We've gone past the size limit. Return the previous item. 401 Chain *Ch = *--I; 402 L.erase(I); 403 return Ch; 404 } 405 406 if ((*I)->getPreferredColor() == PreferredColor) { 407 Chain *Ch = *I; 408 L.erase(I); 409 return Ch; 410 } 411 } 412 413 // Bailout case - just return the first item. 414 Chain *Ch = L.front(); 415 L.erase(L.begin()); 416 return Ch; 417 } 418 419 bool AArch64A57FPLoadBalancing::colorChainSet(std::vector<Chain*> GV, 420 MachineBasicBlock &MBB, 421 int &Parity) { 422 bool Changed = false; 423 DEBUG(dbgs() << "colorChainSet(): #sets=" << GV.size() << "\n"); 424 425 // Sort by descending size order so that we allocate the most important 426 // sets first. 427 // Tie-break equivalent sizes by sorting chains requiring fixups before 428 // those without fixups. The logic here is that we should look at the 429 // chains that we cannot change before we look at those we can, 430 // so the parity counter is updated and we know what color we should 431 // change them to! 432 std::sort(GV.begin(), GV.end(), [](const Chain *G1, const Chain *G2) { 433 if (G1->size() != G2->size()) 434 return G1->size() > G2->size(); 435 return G1->requiresFixup() > G2->requiresFixup(); 436 }); 437 438 Color PreferredColor = Parity < 0 ? Color::Even : Color::Odd; 439 while (Chain *G = getAndEraseNext(PreferredColor, GV)) { 440 // Start off by assuming we'll color to our own preferred color. 441 Color C = PreferredColor; 442 if (Parity == 0) 443 // But if we really don't care, use the chain's preferred color. 444 C = G->getPreferredColor(); 445 446 DEBUG(dbgs() << " - Parity=" << Parity << ", Color=" 447 << ColorNames[(int)C] << "\n"); 448 449 // If we'll need a fixup FMOV, don't bother. Testing has shown that this 450 // happens infrequently and when it does it has at least a 50% chance of 451 // slowing code down instead of speeding it up. 452 if (G->requiresFixup() && C != G->getPreferredColor()) { 453 C = G->getPreferredColor(); 454 DEBUG(dbgs() << " - " << G->str() << " - not worthwhile changing; " 455 "color remains " << ColorNames[(int)C] << "\n"); 456 } 457 458 Changed |= colorChain(G, C, MBB); 459 460 Parity += (C == Color::Even) ? G->size() : -G->size(); 461 PreferredColor = Parity < 0 ? Color::Even : Color::Odd; 462 } 463 464 return Changed; 465 } 466 467 int AArch64A57FPLoadBalancing::scavengeRegister(Chain *G, Color C, 468 MachineBasicBlock &MBB) { 469 RegScavenger RS; 470 RS.enterBasicBlock(&MBB); 471 RS.forward(MachineBasicBlock::iterator(G->getStart())); 472 473 // Can we find an appropriate register that is available throughout the life 474 // of the chain? 475 unsigned RegClassID = G->getStart()->getDesc().OpInfo[0].RegClass; 476 BitVector AvailableRegs = RS.getRegsAvailable(TRI->getRegClass(RegClassID)); 477 for (MachineBasicBlock::iterator I = G->getStart(), E = G->getEnd(); 478 I != E; ++I) { 479 RS.forward(I); 480 AvailableRegs &= RS.getRegsAvailable(TRI->getRegClass(RegClassID)); 481 482 // Remove any registers clobbered by a regmask. 483 for (auto J : I->operands()) { 484 if (J.isRegMask()) 485 AvailableRegs.clearBitsNotInMask(J.getRegMask()); 486 } 487 } 488 489 // Make sure we allocate in-order, to get the cheapest registers first. 490 auto Ord = RCI.getOrder(TRI->getRegClass(RegClassID)); 491 for (auto Reg : Ord) { 492 if (!AvailableRegs[Reg]) 493 continue; 494 if ((C == Color::Even && (Reg % 2) == 0) || 495 (C == Color::Odd && (Reg % 2) == 1)) 496 return Reg; 497 } 498 499 return -1; 500 } 501 502 bool AArch64A57FPLoadBalancing::colorChain(Chain *G, Color C, 503 MachineBasicBlock &MBB) { 504 bool Changed = false; 505 DEBUG(dbgs() << " - colorChain(" << G->str() << ", " 506 << ColorNames[(int)C] << ")\n"); 507 508 // Try and obtain a free register of the right class. Without a register 509 // to play with we cannot continue. 510 int Reg = scavengeRegister(G, C, MBB); 511 if (Reg == -1) { 512 DEBUG(dbgs() << "Scavenging (thus coloring) failed!\n"); 513 return false; 514 } 515 DEBUG(dbgs() << " - Scavenged register: " << TRI->getName(Reg) << "\n"); 516 517 std::map<unsigned, unsigned> Substs; 518 for (MachineBasicBlock::iterator I = G->getStart(), E = G->getEnd(); 519 I != E; ++I) { 520 if (!G->contains(I) && 521 (&*I != G->getKill() || G->isKillImmutable())) 522 continue; 523 524 // I is a member of G, or I is a mutable instruction that kills G. 525 526 std::vector<unsigned> ToErase; 527 for (auto &U : I->operands()) { 528 if (U.isReg() && U.isUse() && Substs.find(U.getReg()) != Substs.end()) { 529 unsigned OrigReg = U.getReg(); 530 U.setReg(Substs[OrigReg]); 531 if (U.isKill()) 532 // Don't erase straight away, because there may be other operands 533 // that also reference this substitution! 534 ToErase.push_back(OrigReg); 535 } else if (U.isRegMask()) { 536 for (auto J : Substs) { 537 if (U.clobbersPhysReg(J.first)) 538 ToErase.push_back(J.first); 539 } 540 } 541 } 542 // Now it's safe to remove the substs identified earlier. 543 for (auto J : ToErase) 544 Substs.erase(J); 545 546 // Only change the def if this isn't the last instruction. 547 if (&*I != G->getKill()) { 548 MachineOperand &MO = I->getOperand(0); 549 550 bool Change = TransformAll || getColor(MO.getReg()) != C; 551 if (G->requiresFixup() && &*I == G->getLast()) 552 Change = false; 553 554 if (Change) { 555 Substs[MO.getReg()] = Reg; 556 MO.setReg(Reg); 557 MRI->setPhysRegUsed(Reg); 558 559 Changed = true; 560 } 561 } 562 } 563 assert(Substs.size() == 0 && "No substitutions should be left active!"); 564 565 if (G->getKill()) { 566 DEBUG(dbgs() << " - Kill instruction seen.\n"); 567 } else { 568 // We didn't have a kill instruction, but we didn't seem to need to change 569 // the destination register anyway. 570 DEBUG(dbgs() << " - Destination register not changed.\n"); 571 } 572 return Changed; 573 } 574 575 void AArch64A57FPLoadBalancing:: 576 scanInstruction(MachineInstr *MI, unsigned Idx, 577 std::map<unsigned, Chain*> &ActiveChains, 578 std::set<std::unique_ptr<Chain>> &AllChains) { 579 // Inspect "MI", updating ActiveChains and AllChains. 580 581 if (isMul(MI)) { 582 583 for (auto &I : MI->uses()) 584 maybeKillChain(I, Idx, ActiveChains); 585 for (auto &I : MI->defs()) 586 maybeKillChain(I, Idx, ActiveChains); 587 588 // Create a new chain. Multiplies don't require forwarding so can go on any 589 // unit. 590 unsigned DestReg = MI->getOperand(0).getReg(); 591 592 DEBUG(dbgs() << "New chain started for register " 593 << TRI->getName(DestReg) << " at " << *MI); 594 595 auto G = llvm::make_unique<Chain>(MI, Idx, getColor(DestReg)); 596 ActiveChains[DestReg] = G.get(); 597 AllChains.insert(std::move(G)); 598 599 } else if (isMla(MI)) { 600 601 // It is beneficial to keep MLAs on the same functional unit as their 602 // accumulator operand. 603 unsigned DestReg = MI->getOperand(0).getReg(); 604 unsigned AccumReg = MI->getOperand(3).getReg(); 605 606 maybeKillChain(MI->getOperand(1), Idx, ActiveChains); 607 maybeKillChain(MI->getOperand(2), Idx, ActiveChains); 608 if (DestReg != AccumReg) 609 maybeKillChain(MI->getOperand(0), Idx, ActiveChains); 610 611 if (ActiveChains.find(AccumReg) != ActiveChains.end()) { 612 DEBUG(dbgs() << "Chain found for accumulator register " 613 << TRI->getName(AccumReg) << " in MI " << *MI); 614 615 // For simplicity we only chain together sequences of MULs/MLAs where the 616 // accumulator register is killed on each instruction. This means we don't 617 // need to track other uses of the registers we want to rewrite. 618 // 619 // FIXME: We could extend to handle the non-kill cases for more coverage. 620 if (MI->getOperand(3).isKill()) { 621 // Add to chain. 622 DEBUG(dbgs() << "Instruction was successfully added to chain.\n"); 623 ActiveChains[AccumReg]->add(MI, Idx, getColor(DestReg)); 624 // Handle cases where the destination is not the same as the accumulator. 625 if (DestReg != AccumReg) { 626 ActiveChains[DestReg] = ActiveChains[AccumReg]; 627 ActiveChains.erase(AccumReg); 628 } 629 return; 630 } 631 632 DEBUG(dbgs() << "Cannot add to chain because accumulator operand wasn't " 633 << "marked <kill>!\n"); 634 maybeKillChain(MI->getOperand(3), Idx, ActiveChains); 635 } 636 637 DEBUG(dbgs() << "Creating new chain for dest register " 638 << TRI->getName(DestReg) << "\n"); 639 auto G = llvm::make_unique<Chain>(MI, Idx, getColor(DestReg)); 640 ActiveChains[DestReg] = G.get(); 641 AllChains.insert(std::move(G)); 642 643 } else { 644 645 // Non-MUL or MLA instruction. Invalidate any chain in the uses or defs 646 // lists. 647 for (auto &I : MI->uses()) 648 maybeKillChain(I, Idx, ActiveChains); 649 for (auto &I : MI->defs()) 650 maybeKillChain(I, Idx, ActiveChains); 651 652 } 653 } 654 655 void AArch64A57FPLoadBalancing:: 656 maybeKillChain(MachineOperand &MO, unsigned Idx, 657 std::map<unsigned, Chain*> &ActiveChains) { 658 // Given an operand and the set of active chains (keyed by register), 659 // determine if a chain should be ended and remove from ActiveChains. 660 MachineInstr *MI = MO.getParent(); 661 662 if (MO.isReg()) { 663 664 // If this is a KILL of a current chain, record it. 665 if (MO.isKill() && ActiveChains.find(MO.getReg()) != ActiveChains.end()) { 666 DEBUG(dbgs() << "Kill seen for chain " << TRI->getName(MO.getReg()) 667 << "\n"); 668 ActiveChains[MO.getReg()]->setKill(MI, Idx, /*Immutable=*/MO.isTied()); 669 } 670 ActiveChains.erase(MO.getReg()); 671 672 } else if (MO.isRegMask()) { 673 674 for (auto I = ActiveChains.begin(), E = ActiveChains.end(); 675 I != E;) { 676 if (MO.clobbersPhysReg(I->first)) { 677 DEBUG(dbgs() << "Kill (regmask) seen for chain " 678 << TRI->getName(I->first) << "\n"); 679 I->second->setKill(MI, Idx, /*Immutable=*/true); 680 ActiveChains.erase(I++); 681 } else 682 ++I; 683 } 684 685 } 686 } 687 688 Color AArch64A57FPLoadBalancing::getColor(unsigned Reg) { 689 if ((TRI->getEncodingValue(Reg) % 2) == 0) 690 return Color::Even; 691 else 692 return Color::Odd; 693 } 694 695 // Factory function used by AArch64TargetMachine to add the pass to the passmanager. 696 FunctionPass *llvm::createAArch64A57FPLoadBalancing() { 697 return new AArch64A57FPLoadBalancing(); 698 } 699