1 //===-- AArch64A57FPLoadBalancing.cpp - Balance FP ops statically on A57---===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // For best-case performance on Cortex-A57, we should try to use a balanced 10 // mix of odd and even D-registers when performing a critical sequence of 11 // independent, non-quadword FP/ASIMD floating-point multiply or 12 // multiply-accumulate operations. 13 // 14 // This pass attempts to detect situations where the register allocation may 15 // adversely affect this load balancing and to change the registers used so as 16 // to better utilize the CPU. 17 // 18 // Ideally we'd just take each multiply or multiply-accumulate in turn and 19 // allocate it alternating even or odd registers. However, multiply-accumulates 20 // are most efficiently performed in the same functional unit as their 21 // accumulation operand. Therefore this pass tries to find maximal sequences 22 // ("Chains") of multiply-accumulates linked via their accumulation operand, 23 // and assign them all the same "color" (oddness/evenness). 24 // 25 // This optimization affects S-register and D-register floating point 26 // multiplies and FMADD/FMAs, as well as vector (floating point only) muls and 27 // FMADD/FMA. Q register instructions (and 128-bit vector instructions) are 28 // not affected. 29 //===----------------------------------------------------------------------===// 30 31 #include "AArch64.h" 32 #include "AArch64InstrInfo.h" 33 #include "AArch64Subtarget.h" 34 #include "llvm/ADT/BitVector.h" 35 #include "llvm/ADT/EquivalenceClasses.h" 36 #include "llvm/CodeGen/MachineFunction.h" 37 #include "llvm/CodeGen/MachineFunctionPass.h" 38 #include "llvm/CodeGen/MachineInstr.h" 39 #include "llvm/CodeGen/MachineInstrBuilder.h" 40 #include "llvm/CodeGen/MachineRegisterInfo.h" 41 #include "llvm/CodeGen/RegisterClassInfo.h" 42 #include "llvm/CodeGen/RegisterScavenging.h" 43 #include "llvm/Support/CommandLine.h" 44 #include "llvm/Support/Debug.h" 45 #include "llvm/Support/raw_ostream.h" 46 using namespace llvm; 47 48 #define DEBUG_TYPE "aarch64-a57-fp-load-balancing" 49 50 // Enforce the algorithm to use the scavenged register even when the original 51 // destination register is the correct color. Used for testing. 52 static cl::opt<bool> 53 TransformAll("aarch64-a57-fp-load-balancing-force-all", 54 cl::desc("Always modify dest registers regardless of color"), 55 cl::init(false), cl::Hidden); 56 57 // Never use the balance information obtained from chains - return a specific 58 // color always. Used for testing. 59 static cl::opt<unsigned> 60 OverrideBalance("aarch64-a57-fp-load-balancing-override", 61 cl::desc("Ignore balance information, always return " 62 "(1: Even, 2: Odd)."), 63 cl::init(0), cl::Hidden); 64 65 //===----------------------------------------------------------------------===// 66 // Helper functions 67 68 // Is the instruction a type of multiply on 64-bit (or 32-bit) FPRs? 69 static bool isMul(MachineInstr *MI) { 70 switch (MI->getOpcode()) { 71 case AArch64::FMULSrr: 72 case AArch64::FNMULSrr: 73 case AArch64::FMULDrr: 74 case AArch64::FNMULDrr: 75 return true; 76 default: 77 return false; 78 } 79 } 80 81 // Is the instruction a type of FP multiply-accumulate on 64-bit (or 32-bit) FPRs? 82 static bool isMla(MachineInstr *MI) { 83 switch (MI->getOpcode()) { 84 case AArch64::FMSUBSrrr: 85 case AArch64::FMADDSrrr: 86 case AArch64::FNMSUBSrrr: 87 case AArch64::FNMADDSrrr: 88 case AArch64::FMSUBDrrr: 89 case AArch64::FMADDDrrr: 90 case AArch64::FNMSUBDrrr: 91 case AArch64::FNMADDDrrr: 92 return true; 93 default: 94 return false; 95 } 96 } 97 98 //===----------------------------------------------------------------------===// 99 100 namespace { 101 /// A "color", which is either even or odd. Yes, these aren't really colors 102 /// but the algorithm is conceptually doing two-color graph coloring. 103 enum class Color { Even, Odd }; 104 #ifndef NDEBUG 105 static const char *ColorNames[2] = { "Even", "Odd" }; 106 #endif 107 108 class Chain; 109 110 class AArch64A57FPLoadBalancing : public MachineFunctionPass { 111 MachineRegisterInfo *MRI; 112 const TargetRegisterInfo *TRI; 113 RegisterClassInfo RCI; 114 115 public: 116 static char ID; 117 explicit AArch64A57FPLoadBalancing() : MachineFunctionPass(ID) { 118 initializeAArch64A57FPLoadBalancingPass(*PassRegistry::getPassRegistry()); 119 } 120 121 bool runOnMachineFunction(MachineFunction &F) override; 122 123 MachineFunctionProperties getRequiredProperties() const override { 124 return MachineFunctionProperties().set( 125 MachineFunctionProperties::Property::NoVRegs); 126 } 127 128 StringRef getPassName() const override { 129 return "A57 FP Anti-dependency breaker"; 130 } 131 132 void getAnalysisUsage(AnalysisUsage &AU) const override { 133 AU.setPreservesCFG(); 134 MachineFunctionPass::getAnalysisUsage(AU); 135 } 136 137 private: 138 bool runOnBasicBlock(MachineBasicBlock &MBB); 139 bool colorChainSet(std::vector<Chain*> GV, MachineBasicBlock &MBB, 140 int &Balance); 141 bool colorChain(Chain *G, Color C, MachineBasicBlock &MBB); 142 int scavengeRegister(Chain *G, Color C, MachineBasicBlock &MBB); 143 void scanInstruction(MachineInstr *MI, unsigned Idx, 144 std::map<unsigned, Chain*> &Active, 145 std::vector<std::unique_ptr<Chain>> &AllChains); 146 void maybeKillChain(MachineOperand &MO, unsigned Idx, 147 std::map<unsigned, Chain*> &RegChains); 148 Color getColor(unsigned Register); 149 Chain *getAndEraseNext(Color PreferredColor, std::vector<Chain*> &L); 150 }; 151 } 152 153 char AArch64A57FPLoadBalancing::ID = 0; 154 155 INITIALIZE_PASS_BEGIN(AArch64A57FPLoadBalancing, DEBUG_TYPE, 156 "AArch64 A57 FP Load-Balancing", false, false) 157 INITIALIZE_PASS_END(AArch64A57FPLoadBalancing, DEBUG_TYPE, 158 "AArch64 A57 FP Load-Balancing", false, false) 159 160 namespace { 161 /// A Chain is a sequence of instructions that are linked together by 162 /// an accumulation operand. For example: 163 /// 164 /// fmul def d0, ? 165 /// fmla def d1, ?, ?, killed d0 166 /// fmla def d2, ?, ?, killed d1 167 /// 168 /// There may be other instructions interleaved in the sequence that 169 /// do not belong to the chain. These other instructions must not use 170 /// the "chain" register at any point. 171 /// 172 /// We currently only support chains where the "chain" operand is killed 173 /// at each link in the chain for simplicity. 174 /// A chain has three important instructions - Start, Last and Kill. 175 /// * The start instruction is the first instruction in the chain. 176 /// * Last is the final instruction in the chain. 177 /// * Kill may or may not be defined. If defined, Kill is the instruction 178 /// where the outgoing value of the Last instruction is killed. 179 /// This information is important as if we know the outgoing value is 180 /// killed with no intervening uses, we can safely change its register. 181 /// 182 /// Without a kill instruction, we must assume the outgoing value escapes 183 /// beyond our model and either must not change its register or must 184 /// create a fixup FMOV to keep the old register value consistent. 185 /// 186 class Chain { 187 public: 188 /// The important (marker) instructions. 189 MachineInstr *StartInst, *LastInst, *KillInst; 190 /// The index, from the start of the basic block, that each marker 191 /// appears. These are stored so we can do quick interval tests. 192 unsigned StartInstIdx, LastInstIdx, KillInstIdx; 193 /// All instructions in the chain. 194 std::set<MachineInstr*> Insts; 195 /// True if KillInst cannot be modified. If this is true, 196 /// we cannot change LastInst's outgoing register. 197 /// This will be true for tied values and regmasks. 198 bool KillIsImmutable; 199 /// The "color" of LastInst. This will be the preferred chain color, 200 /// as changing intermediate nodes is easy but changing the last 201 /// instruction can be more tricky. 202 Color LastColor; 203 204 Chain(MachineInstr *MI, unsigned Idx, Color C) 205 : StartInst(MI), LastInst(MI), KillInst(nullptr), 206 StartInstIdx(Idx), LastInstIdx(Idx), KillInstIdx(0), 207 LastColor(C) { 208 Insts.insert(MI); 209 } 210 211 /// Add a new instruction into the chain. The instruction's dest operand 212 /// has the given color. 213 void add(MachineInstr *MI, unsigned Idx, Color C) { 214 LastInst = MI; 215 LastInstIdx = Idx; 216 LastColor = C; 217 assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) && 218 "Chain: broken invariant. A Chain can only be killed after its last " 219 "def"); 220 221 Insts.insert(MI); 222 } 223 224 /// Return true if MI is a member of the chain. 225 bool contains(MachineInstr &MI) { return Insts.count(&MI) > 0; } 226 227 /// Return the number of instructions in the chain. 228 unsigned size() const { 229 return Insts.size(); 230 } 231 232 /// Inform the chain that its last active register (the dest register of 233 /// LastInst) is killed by MI with no intervening uses or defs. 234 void setKill(MachineInstr *MI, unsigned Idx, bool Immutable) { 235 KillInst = MI; 236 KillInstIdx = Idx; 237 KillIsImmutable = Immutable; 238 assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) && 239 "Chain: broken invariant. A Chain can only be killed after its last " 240 "def"); 241 } 242 243 /// Return the first instruction in the chain. 244 MachineInstr *getStart() const { return StartInst; } 245 /// Return the last instruction in the chain. 246 MachineInstr *getLast() const { return LastInst; } 247 /// Return the "kill" instruction (as set with setKill()) or NULL. 248 MachineInstr *getKill() const { return KillInst; } 249 /// Return an instruction that can be used as an iterator for the end 250 /// of the chain. This is the maximum of KillInst (if set) and LastInst. 251 MachineBasicBlock::iterator end() const { 252 return ++MachineBasicBlock::iterator(KillInst ? KillInst : LastInst); 253 } 254 MachineBasicBlock::iterator begin() const { return getStart(); } 255 256 /// Can the Kill instruction (assuming one exists) be modified? 257 bool isKillImmutable() const { return KillIsImmutable; } 258 259 /// Return the preferred color of this chain. 260 Color getPreferredColor() { 261 if (OverrideBalance != 0) 262 return OverrideBalance == 1 ? Color::Even : Color::Odd; 263 return LastColor; 264 } 265 266 /// Return true if this chain (StartInst..KillInst) overlaps with Other. 267 bool rangeOverlapsWith(const Chain &Other) const { 268 unsigned End = KillInst ? KillInstIdx : LastInstIdx; 269 unsigned OtherEnd = Other.KillInst ? 270 Other.KillInstIdx : Other.LastInstIdx; 271 272 return StartInstIdx <= OtherEnd && Other.StartInstIdx <= End; 273 } 274 275 /// Return true if this chain starts before Other. 276 bool startsBefore(const Chain *Other) const { 277 return StartInstIdx < Other->StartInstIdx; 278 } 279 280 /// Return true if the group will require a fixup MOV at the end. 281 bool requiresFixup() const { 282 return (getKill() && isKillImmutable()) || !getKill(); 283 } 284 285 /// Return a simple string representation of the chain. 286 std::string str() const { 287 std::string S; 288 raw_string_ostream OS(S); 289 290 OS << "{"; 291 StartInst->print(OS, /* SkipOpers= */true); 292 OS << " -> "; 293 LastInst->print(OS, /* SkipOpers= */true); 294 if (KillInst) { 295 OS << " (kill @ "; 296 KillInst->print(OS, /* SkipOpers= */true); 297 OS << ")"; 298 } 299 OS << "}"; 300 301 return OS.str(); 302 } 303 304 }; 305 306 } // end anonymous namespace 307 308 //===----------------------------------------------------------------------===// 309 310 bool AArch64A57FPLoadBalancing::runOnMachineFunction(MachineFunction &F) { 311 if (skipFunction(F.getFunction())) 312 return false; 313 314 if (!F.getSubtarget<AArch64Subtarget>().balanceFPOps()) 315 return false; 316 317 bool Changed = false; 318 LLVM_DEBUG(dbgs() << "***** AArch64A57FPLoadBalancing *****\n"); 319 320 MRI = &F.getRegInfo(); 321 TRI = F.getRegInfo().getTargetRegisterInfo(); 322 RCI.runOnMachineFunction(F); 323 324 for (auto &MBB : F) { 325 Changed |= runOnBasicBlock(MBB); 326 } 327 328 return Changed; 329 } 330 331 bool AArch64A57FPLoadBalancing::runOnBasicBlock(MachineBasicBlock &MBB) { 332 bool Changed = false; 333 LLVM_DEBUG(dbgs() << "Running on MBB: " << MBB 334 << " - scanning instructions...\n"); 335 336 // First, scan the basic block producing a set of chains. 337 338 // The currently "active" chains - chains that can be added to and haven't 339 // been killed yet. This is keyed by register - all chains can only have one 340 // "link" register between each inst in the chain. 341 std::map<unsigned, Chain*> ActiveChains; 342 std::vector<std::unique_ptr<Chain>> AllChains; 343 unsigned Idx = 0; 344 for (auto &MI : MBB) 345 scanInstruction(&MI, Idx++, ActiveChains, AllChains); 346 347 LLVM_DEBUG(dbgs() << "Scan complete, " << AllChains.size() 348 << " chains created.\n"); 349 350 // Group the chains into disjoint sets based on their liveness range. This is 351 // a poor-man's version of graph coloring. Ideally we'd create an interference 352 // graph and perform full-on graph coloring on that, but; 353 // (a) That's rather heavyweight for only two colors. 354 // (b) We expect multiple disjoint interference regions - in practice the live 355 // range of chains is quite small and they are clustered between loads 356 // and stores. 357 EquivalenceClasses<Chain*> EC; 358 for (auto &I : AllChains) 359 EC.insert(I.get()); 360 361 for (auto &I : AllChains) 362 for (auto &J : AllChains) 363 if (I != J && I->rangeOverlapsWith(*J)) 364 EC.unionSets(I.get(), J.get()); 365 LLVM_DEBUG(dbgs() << "Created " << EC.getNumClasses() << " disjoint sets.\n"); 366 367 // Now we assume that every member of an equivalence class interferes 368 // with every other member of that class, and with no members of other classes. 369 370 // Convert the EquivalenceClasses to a simpler set of sets. 371 std::vector<std::vector<Chain*> > V; 372 for (auto I = EC.begin(), E = EC.end(); I != E; ++I) { 373 std::vector<Chain*> Cs(EC.member_begin(I), EC.member_end()); 374 if (Cs.empty()) continue; 375 V.push_back(std::move(Cs)); 376 } 377 378 // Now we have a set of sets, order them by start address so 379 // we can iterate over them sequentially. 380 llvm::sort(V.begin(), V.end(), 381 [](const std::vector<Chain*> &A, 382 const std::vector<Chain*> &B) { 383 return A.front()->startsBefore(B.front()); 384 }); 385 386 // As we only have two colors, we can track the global (BB-level) balance of 387 // odds versus evens. We aim to keep this near zero to keep both execution 388 // units fed. 389 // Positive means we're even-heavy, negative we're odd-heavy. 390 // 391 // FIXME: If chains have interdependencies, for example: 392 // mul r0, r1, r2 393 // mul r3, r0, r1 394 // We do not model this and may color each one differently, assuming we'll 395 // get ILP when we obviously can't. This hasn't been seen to be a problem 396 // in practice so far, so we simplify the algorithm by ignoring it. 397 int Parity = 0; 398 399 for (auto &I : V) 400 Changed |= colorChainSet(std::move(I), MBB, Parity); 401 402 return Changed; 403 } 404 405 Chain *AArch64A57FPLoadBalancing::getAndEraseNext(Color PreferredColor, 406 std::vector<Chain*> &L) { 407 if (L.empty()) 408 return nullptr; 409 410 // We try and get the best candidate from L to color next, given that our 411 // preferred color is "PreferredColor". L is ordered from larger to smaller 412 // chains. It is beneficial to color the large chains before the small chains, 413 // but if we can't find a chain of the maximum length with the preferred color, 414 // we fuzz the size and look for slightly smaller chains before giving up and 415 // returning a chain that must be recolored. 416 417 // FIXME: Does this need to be configurable? 418 const unsigned SizeFuzz = 1; 419 unsigned MinSize = L.front()->size() - SizeFuzz; 420 for (auto I = L.begin(), E = L.end(); I != E; ++I) { 421 if ((*I)->size() <= MinSize) { 422 // We've gone past the size limit. Return the previous item. 423 Chain *Ch = *--I; 424 L.erase(I); 425 return Ch; 426 } 427 428 if ((*I)->getPreferredColor() == PreferredColor) { 429 Chain *Ch = *I; 430 L.erase(I); 431 return Ch; 432 } 433 } 434 435 // Bailout case - just return the first item. 436 Chain *Ch = L.front(); 437 L.erase(L.begin()); 438 return Ch; 439 } 440 441 bool AArch64A57FPLoadBalancing::colorChainSet(std::vector<Chain*> GV, 442 MachineBasicBlock &MBB, 443 int &Parity) { 444 bool Changed = false; 445 LLVM_DEBUG(dbgs() << "colorChainSet(): #sets=" << GV.size() << "\n"); 446 447 // Sort by descending size order so that we allocate the most important 448 // sets first. 449 // Tie-break equivalent sizes by sorting chains requiring fixups before 450 // those without fixups. The logic here is that we should look at the 451 // chains that we cannot change before we look at those we can, 452 // so the parity counter is updated and we know what color we should 453 // change them to! 454 // Final tie-break with instruction order so pass output is stable (i.e. not 455 // dependent on malloc'd pointer values). 456 llvm::sort(GV.begin(), GV.end(), [](const Chain *G1, const Chain *G2) { 457 if (G1->size() != G2->size()) 458 return G1->size() > G2->size(); 459 if (G1->requiresFixup() != G2->requiresFixup()) 460 return G1->requiresFixup() > G2->requiresFixup(); 461 // Make sure startsBefore() produces a stable final order. 462 assert((G1 == G2 || (G1->startsBefore(G2) ^ G2->startsBefore(G1))) && 463 "Starts before not total order!"); 464 return G1->startsBefore(G2); 465 }); 466 467 Color PreferredColor = Parity < 0 ? Color::Even : Color::Odd; 468 while (Chain *G = getAndEraseNext(PreferredColor, GV)) { 469 // Start off by assuming we'll color to our own preferred color. 470 Color C = PreferredColor; 471 if (Parity == 0) 472 // But if we really don't care, use the chain's preferred color. 473 C = G->getPreferredColor(); 474 475 LLVM_DEBUG(dbgs() << " - Parity=" << Parity 476 << ", Color=" << ColorNames[(int)C] << "\n"); 477 478 // If we'll need a fixup FMOV, don't bother. Testing has shown that this 479 // happens infrequently and when it does it has at least a 50% chance of 480 // slowing code down instead of speeding it up. 481 if (G->requiresFixup() && C != G->getPreferredColor()) { 482 C = G->getPreferredColor(); 483 LLVM_DEBUG(dbgs() << " - " << G->str() 484 << " - not worthwhile changing; " 485 "color remains " 486 << ColorNames[(int)C] << "\n"); 487 } 488 489 Changed |= colorChain(G, C, MBB); 490 491 Parity += (C == Color::Even) ? G->size() : -G->size(); 492 PreferredColor = Parity < 0 ? Color::Even : Color::Odd; 493 } 494 495 return Changed; 496 } 497 498 int AArch64A57FPLoadBalancing::scavengeRegister(Chain *G, Color C, 499 MachineBasicBlock &MBB) { 500 // Can we find an appropriate register that is available throughout the life 501 // of the chain? Simulate liveness backwards until the end of the chain. 502 LiveRegUnits Units(*TRI); 503 Units.addLiveOuts(MBB); 504 MachineBasicBlock::iterator I = MBB.end(); 505 MachineBasicBlock::iterator ChainEnd = G->end(); 506 while (I != ChainEnd) { 507 --I; 508 Units.stepBackward(*I); 509 } 510 511 // Check which register units are alive throughout the chain. 512 MachineBasicBlock::iterator ChainBegin = G->begin(); 513 assert(ChainBegin != ChainEnd && "Chain should contain instructions"); 514 do { 515 --I; 516 Units.accumulate(*I); 517 } while (I != ChainBegin); 518 519 // Make sure we allocate in-order, to get the cheapest registers first. 520 unsigned RegClassID = ChainBegin->getDesc().OpInfo[0].RegClass; 521 auto Ord = RCI.getOrder(TRI->getRegClass(RegClassID)); 522 for (auto Reg : Ord) { 523 if (!Units.available(Reg)) 524 continue; 525 if (C == getColor(Reg)) 526 return Reg; 527 } 528 529 return -1; 530 } 531 532 bool AArch64A57FPLoadBalancing::colorChain(Chain *G, Color C, 533 MachineBasicBlock &MBB) { 534 bool Changed = false; 535 LLVM_DEBUG(dbgs() << " - colorChain(" << G->str() << ", " 536 << ColorNames[(int)C] << ")\n"); 537 538 // Try and obtain a free register of the right class. Without a register 539 // to play with we cannot continue. 540 int Reg = scavengeRegister(G, C, MBB); 541 if (Reg == -1) { 542 LLVM_DEBUG(dbgs() << "Scavenging (thus coloring) failed!\n"); 543 return false; 544 } 545 LLVM_DEBUG(dbgs() << " - Scavenged register: " << printReg(Reg, TRI) << "\n"); 546 547 std::map<unsigned, unsigned> Substs; 548 for (MachineInstr &I : *G) { 549 if (!G->contains(I) && (&I != G->getKill() || G->isKillImmutable())) 550 continue; 551 552 // I is a member of G, or I is a mutable instruction that kills G. 553 554 std::vector<unsigned> ToErase; 555 for (auto &U : I.operands()) { 556 if (U.isReg() && U.isUse() && Substs.find(U.getReg()) != Substs.end()) { 557 unsigned OrigReg = U.getReg(); 558 U.setReg(Substs[OrigReg]); 559 if (U.isKill()) 560 // Don't erase straight away, because there may be other operands 561 // that also reference this substitution! 562 ToErase.push_back(OrigReg); 563 } else if (U.isRegMask()) { 564 for (auto J : Substs) { 565 if (U.clobbersPhysReg(J.first)) 566 ToErase.push_back(J.first); 567 } 568 } 569 } 570 // Now it's safe to remove the substs identified earlier. 571 for (auto J : ToErase) 572 Substs.erase(J); 573 574 // Only change the def if this isn't the last instruction. 575 if (&I != G->getKill()) { 576 MachineOperand &MO = I.getOperand(0); 577 578 bool Change = TransformAll || getColor(MO.getReg()) != C; 579 if (G->requiresFixup() && &I == G->getLast()) 580 Change = false; 581 582 if (Change) { 583 Substs[MO.getReg()] = Reg; 584 MO.setReg(Reg); 585 586 Changed = true; 587 } 588 } 589 } 590 assert(Substs.size() == 0 && "No substitutions should be left active!"); 591 592 if (G->getKill()) { 593 LLVM_DEBUG(dbgs() << " - Kill instruction seen.\n"); 594 } else { 595 // We didn't have a kill instruction, but we didn't seem to need to change 596 // the destination register anyway. 597 LLVM_DEBUG(dbgs() << " - Destination register not changed.\n"); 598 } 599 return Changed; 600 } 601 602 void AArch64A57FPLoadBalancing::scanInstruction( 603 MachineInstr *MI, unsigned Idx, std::map<unsigned, Chain *> &ActiveChains, 604 std::vector<std::unique_ptr<Chain>> &AllChains) { 605 // Inspect "MI", updating ActiveChains and AllChains. 606 607 if (isMul(MI)) { 608 609 for (auto &I : MI->uses()) 610 maybeKillChain(I, Idx, ActiveChains); 611 for (auto &I : MI->defs()) 612 maybeKillChain(I, Idx, ActiveChains); 613 614 // Create a new chain. Multiplies don't require forwarding so can go on any 615 // unit. 616 unsigned DestReg = MI->getOperand(0).getReg(); 617 618 LLVM_DEBUG(dbgs() << "New chain started for register " 619 << printReg(DestReg, TRI) << " at " << *MI); 620 621 auto G = llvm::make_unique<Chain>(MI, Idx, getColor(DestReg)); 622 ActiveChains[DestReg] = G.get(); 623 AllChains.push_back(std::move(G)); 624 625 } else if (isMla(MI)) { 626 627 // It is beneficial to keep MLAs on the same functional unit as their 628 // accumulator operand. 629 unsigned DestReg = MI->getOperand(0).getReg(); 630 unsigned AccumReg = MI->getOperand(3).getReg(); 631 632 maybeKillChain(MI->getOperand(1), Idx, ActiveChains); 633 maybeKillChain(MI->getOperand(2), Idx, ActiveChains); 634 if (DestReg != AccumReg) 635 maybeKillChain(MI->getOperand(0), Idx, ActiveChains); 636 637 if (ActiveChains.find(AccumReg) != ActiveChains.end()) { 638 LLVM_DEBUG(dbgs() << "Chain found for accumulator register " 639 << printReg(AccumReg, TRI) << " in MI " << *MI); 640 641 // For simplicity we only chain together sequences of MULs/MLAs where the 642 // accumulator register is killed on each instruction. This means we don't 643 // need to track other uses of the registers we want to rewrite. 644 // 645 // FIXME: We could extend to handle the non-kill cases for more coverage. 646 if (MI->getOperand(3).isKill()) { 647 // Add to chain. 648 LLVM_DEBUG(dbgs() << "Instruction was successfully added to chain.\n"); 649 ActiveChains[AccumReg]->add(MI, Idx, getColor(DestReg)); 650 // Handle cases where the destination is not the same as the accumulator. 651 if (DestReg != AccumReg) { 652 ActiveChains[DestReg] = ActiveChains[AccumReg]; 653 ActiveChains.erase(AccumReg); 654 } 655 return; 656 } 657 658 LLVM_DEBUG( 659 dbgs() << "Cannot add to chain because accumulator operand wasn't " 660 << "marked <kill>!\n"); 661 maybeKillChain(MI->getOperand(3), Idx, ActiveChains); 662 } 663 664 LLVM_DEBUG(dbgs() << "Creating new chain for dest register " 665 << printReg(DestReg, TRI) << "\n"); 666 auto G = llvm::make_unique<Chain>(MI, Idx, getColor(DestReg)); 667 ActiveChains[DestReg] = G.get(); 668 AllChains.push_back(std::move(G)); 669 670 } else { 671 672 // Non-MUL or MLA instruction. Invalidate any chain in the uses or defs 673 // lists. 674 for (auto &I : MI->uses()) 675 maybeKillChain(I, Idx, ActiveChains); 676 for (auto &I : MI->defs()) 677 maybeKillChain(I, Idx, ActiveChains); 678 679 } 680 } 681 682 void AArch64A57FPLoadBalancing:: 683 maybeKillChain(MachineOperand &MO, unsigned Idx, 684 std::map<unsigned, Chain*> &ActiveChains) { 685 // Given an operand and the set of active chains (keyed by register), 686 // determine if a chain should be ended and remove from ActiveChains. 687 MachineInstr *MI = MO.getParent(); 688 689 if (MO.isReg()) { 690 691 // If this is a KILL of a current chain, record it. 692 if (MO.isKill() && ActiveChains.find(MO.getReg()) != ActiveChains.end()) { 693 LLVM_DEBUG(dbgs() << "Kill seen for chain " << printReg(MO.getReg(), TRI) 694 << "\n"); 695 ActiveChains[MO.getReg()]->setKill(MI, Idx, /*Immutable=*/MO.isTied()); 696 } 697 ActiveChains.erase(MO.getReg()); 698 699 } else if (MO.isRegMask()) { 700 701 for (auto I = ActiveChains.begin(), E = ActiveChains.end(); 702 I != E;) { 703 if (MO.clobbersPhysReg(I->first)) { 704 LLVM_DEBUG(dbgs() << "Kill (regmask) seen for chain " 705 << printReg(I->first, TRI) << "\n"); 706 I->second->setKill(MI, Idx, /*Immutable=*/true); 707 ActiveChains.erase(I++); 708 } else 709 ++I; 710 } 711 712 } 713 } 714 715 Color AArch64A57FPLoadBalancing::getColor(unsigned Reg) { 716 if ((TRI->getEncodingValue(Reg) % 2) == 0) 717 return Color::Even; 718 else 719 return Color::Odd; 720 } 721 722 // Factory function used by AArch64TargetMachine to add the pass to the passmanager. 723 FunctionPass *llvm::createAArch64A57FPLoadBalancing() { 724 return new AArch64A57FPLoadBalancing(); 725 } 726