1 //===-- AArch64A53Fix835769.cpp -------------------------------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // This pass changes code to work around Cortex-A53 erratum 835769. 10 // It works around it by inserting a nop instruction in code sequences that 11 // in some circumstances may trigger the erratum. 12 // It inserts a nop instruction between a sequence of the following 2 classes 13 // of instructions: 14 // instr 1: mem-instr (including loads, stores and prefetches). 15 // instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers. 16 //===----------------------------------------------------------------------===// 17 18 #include "AArch64.h" 19 #include "llvm/ADT/Statistic.h" 20 #include "llvm/CodeGen/MachineFunction.h" 21 #include "llvm/CodeGen/MachineFunctionPass.h" 22 #include "llvm/CodeGen/MachineInstr.h" 23 #include "llvm/CodeGen/MachineInstrBuilder.h" 24 #include "llvm/CodeGen/MachineRegisterInfo.h" 25 #include "llvm/Support/CommandLine.h" 26 #include "llvm/Support/Debug.h" 27 #include "llvm/Support/raw_ostream.h" 28 #include "llvm/Target/TargetInstrInfo.h" 29 30 using namespace llvm; 31 32 #define DEBUG_TYPE "aarch64-fix-cortex-a53-835769" 33 34 STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769"); 35 36 //===----------------------------------------------------------------------===// 37 // Helper functions 38 39 // Is the instruction a match for the instruction that comes first in the 40 // sequence of instructions that can trigger the erratum? 41 static bool isFirstInstructionInSequence(MachineInstr *MI) { 42 // Must return true if this instruction is a load, a store or a prefetch. 43 switch (MI->getOpcode()) { 44 case AArch64::PRFMl: 45 case AArch64::PRFMroW: 46 case AArch64::PRFMroX: 47 case AArch64::PRFMui: 48 case AArch64::PRFUMi: 49 return true; 50 default: 51 return (MI->mayLoad() || MI->mayStore()); 52 } 53 } 54 55 // Is the instruction a match for the instruction that comes second in the 56 // sequence that can trigger the erratum? 57 static bool isSecondInstructionInSequence(MachineInstr *MI) { 58 // Must return true for non-SIMD integer multiply-accumulates, writing 59 // to a 64-bit register. 60 switch (MI->getOpcode()) { 61 // Erratum cannot be triggered when the destination register is 32 bits, 62 // therefore only include the following. 63 case AArch64::MSUBXrrr: 64 case AArch64::MADDXrrr: 65 case AArch64::SMADDLrrr: 66 case AArch64::SMSUBLrrr: 67 case AArch64::UMADDLrrr: 68 case AArch64::UMSUBLrrr: 69 // Erratum can only be triggered by multiply-adds, not by regular 70 // non-accumulating multiplies, i.e. when Ra=XZR='11111' 71 return MI->getOperand(3).getReg() != AArch64::XZR; 72 default: 73 return false; 74 } 75 } 76 77 78 //===----------------------------------------------------------------------===// 79 80 namespace { 81 class AArch64A53Fix835769 : public MachineFunctionPass { 82 const TargetInstrInfo *TII; 83 84 public: 85 static char ID; 86 explicit AArch64A53Fix835769() : MachineFunctionPass(ID) {} 87 88 bool runOnMachineFunction(MachineFunction &F) override; 89 90 const char *getPassName() const override { 91 return "Workaround A53 erratum 835769 pass"; 92 } 93 94 void getAnalysisUsage(AnalysisUsage &AU) const override { 95 AU.setPreservesCFG(); 96 MachineFunctionPass::getAnalysisUsage(AU); 97 } 98 99 private: 100 bool runOnBasicBlock(MachineBasicBlock &MBB); 101 }; 102 char AArch64A53Fix835769::ID = 0; 103 104 } // end anonymous namespace 105 106 //===----------------------------------------------------------------------===// 107 108 bool 109 AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) { 110 DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n"); 111 bool Changed = false; 112 TII = F.getSubtarget().getInstrInfo(); 113 114 for (auto &MBB : F) { 115 Changed |= runOnBasicBlock(MBB); 116 } 117 return Changed; 118 } 119 120 // Return the block that was fallen through to get to MBB, if any, 121 // otherwise nullptr. 122 static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB, 123 const TargetInstrInfo *TII) { 124 // Get the previous machine basic block in the function. 125 MachineFunction::iterator MBBI = *MBB; 126 127 // Can't go off top of function. 128 if (MBBI == MBB->getParent()->begin()) 129 return nullptr; 130 131 MachineBasicBlock *TBB = nullptr, *FBB = nullptr; 132 SmallVector<MachineOperand, 2> Cond; 133 134 MachineBasicBlock *PrevBB = std::prev(MBBI); 135 for (MachineBasicBlock *S : MBB->predecessors()) 136 if (S == PrevBB && !TII->AnalyzeBranch(*PrevBB, TBB, FBB, Cond) && 137 !TBB && !FBB) 138 return S; 139 140 return nullptr; 141 } 142 143 // Iterate through fallen through blocks trying to find a previous non-pseudo if 144 // there is one, otherwise return nullptr. Only look for instructions in 145 // previous blocks, not the current block, since we only use this to look at 146 // previous blocks. 147 static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB, 148 const TargetInstrInfo *TII) { 149 MachineBasicBlock *FMBB = &MBB; 150 151 // If there is no non-pseudo in the current block, loop back around and try 152 // the previous block (if there is one). 153 while ((FMBB = getBBFallenThrough(FMBB, TII))) { 154 for (auto I = FMBB->rbegin(), E = FMBB->rend(); I != E; ++I) { 155 if (!I->isPseudo()) 156 return &*I; 157 } 158 } 159 160 // There was no previous non-pseudo in the fallen through blocks 161 return nullptr; 162 } 163 164 static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI, 165 const TargetInstrInfo *TII) { 166 // If we are the first instruction of the block, put the NOP at the end of 167 // the previous fallthrough block 168 if (MI == &MBB.front()) { 169 MachineInstr *I = getLastNonPseudo(MBB, TII); 170 assert(I && "Expected instruction"); 171 DebugLoc DL = I->getDebugLoc(); 172 BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0); 173 } 174 else { 175 DebugLoc DL = MI->getDebugLoc(); 176 BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0); 177 } 178 179 ++NumNopsAdded; 180 } 181 182 bool 183 AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) { 184 bool Changed = false; 185 DEBUG(dbgs() << "Running on MBB: " << MBB << " - scanning instructions...\n"); 186 187 // First, scan the basic block, looking for a sequence of 2 instructions 188 // that match the conditions under which the erratum may trigger. 189 190 // List of terminating instructions in matching sequences 191 std::vector<MachineInstr*> Sequences; 192 unsigned Idx = 0; 193 MachineInstr *PrevInstr = nullptr; 194 195 // Try and find the last non-pseudo instruction in any fallen through blocks, 196 // if there isn't one, then we use nullptr to represent that. 197 PrevInstr = getLastNonPseudo(MBB, TII); 198 199 for (auto &MI : MBB) { 200 MachineInstr *CurrInstr = &MI; 201 DEBUG(dbgs() << " Examining: " << MI); 202 if (PrevInstr) { 203 DEBUG(dbgs() << " PrevInstr: " << *PrevInstr 204 << " CurrInstr: " << *CurrInstr 205 << " isFirstInstructionInSequence(PrevInstr): " 206 << isFirstInstructionInSequence(PrevInstr) << "\n" 207 << " isSecondInstructionInSequence(CurrInstr): " 208 << isSecondInstructionInSequence(CurrInstr) << "\n"); 209 if (isFirstInstructionInSequence(PrevInstr) && 210 isSecondInstructionInSequence(CurrInstr)) { 211 DEBUG(dbgs() << " ** pattern found at Idx " << Idx << "!\n"); 212 Sequences.push_back(CurrInstr); 213 } 214 } 215 if (!CurrInstr->isPseudo()) 216 PrevInstr = CurrInstr; 217 ++Idx; 218 } 219 220 DEBUG(dbgs() << "Scan complete, "<< Sequences.size() 221 << " occurences of pattern found.\n"); 222 223 // Then update the basic block, inserting nops between the detected sequences. 224 for (auto &MI : Sequences) { 225 Changed = true; 226 insertNopBeforeInstruction(MBB, MI, TII); 227 } 228 229 return Changed; 230 } 231 232 // Factory function used by AArch64TargetMachine to add the pass to 233 // the passmanager. 234 FunctionPass *llvm::createAArch64A53Fix835769() { 235 return new AArch64A53Fix835769(); 236 } 237