Home | History | Annotate | Download | only in AArch64 
      1 //===-- AArch64A53Fix835769.cpp -------------------------------------------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 // This pass changes code to work around Cortex-A53 erratum 835769.
     10 // It works around it by inserting a nop instruction in code sequences that
     11 // in some circumstances may trigger the erratum.
     12 // It inserts a nop instruction between a sequence of the following 2 classes
     13 // of instructions:
     14 // instr 1: mem-instr (including loads, stores and prefetches).
     15 // instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers.
     16 //===----------------------------------------------------------------------===//
     17 
     18 #include "AArch64.h"
     19 #include "llvm/ADT/Statistic.h"
     20 #include "llvm/CodeGen/MachineFunction.h"
     21 #include "llvm/CodeGen/MachineFunctionPass.h"
     22 #include "llvm/CodeGen/MachineInstr.h"
     23 #include "llvm/CodeGen/MachineInstrBuilder.h"
     24 #include "llvm/CodeGen/MachineRegisterInfo.h"
     25 #include "llvm/Support/CommandLine.h"
     26 #include "llvm/Support/Debug.h"
     27 #include "llvm/Support/raw_ostream.h"
     28 #include "llvm/Target/TargetInstrInfo.h"
     29 
     30 using namespace llvm;
     31 
     32 #define DEBUG_TYPE "aarch64-fix-cortex-a53-835769"
     33 
     34 STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769");
     35 
     36 //===----------------------------------------------------------------------===//
     37 // Helper functions
     38 
     39 // Is the instruction a match for the instruction that comes first in the
     40 // sequence of instructions that can trigger the erratum?
     41 static bool isFirstInstructionInSequence(MachineInstr *MI) {
     42   // Must return true if this instruction is a load, a store or a prefetch.
     43   switch (MI->getOpcode()) {
     44   case AArch64::PRFMl:
     45   case AArch64::PRFMroW:
     46   case AArch64::PRFMroX:
     47   case AArch64::PRFMui:
     48   case AArch64::PRFUMi:
     49     return true;
     50   default:
     51     return MI->mayLoadOrStore();
     52   }
     53 }
     54 
     55 // Is the instruction a match for the instruction that comes second in the
     56 // sequence that can trigger the erratum?
     57 static bool isSecondInstructionInSequence(MachineInstr *MI) {
     58   // Must return true for non-SIMD integer multiply-accumulates, writing
     59   // to a 64-bit register.
     60   switch (MI->getOpcode()) {
     61   // Erratum cannot be triggered when the destination register is 32 bits,
     62   // therefore only include the following.
     63   case AArch64::MSUBXrrr:
     64   case AArch64::MADDXrrr:
     65   case AArch64::SMADDLrrr:
     66   case AArch64::SMSUBLrrr:
     67   case AArch64::UMADDLrrr:
     68   case AArch64::UMSUBLrrr:
     69     // Erratum can only be triggered by multiply-adds, not by regular
     70     // non-accumulating multiplies, i.e. when Ra=XZR='11111'
     71     return MI->getOperand(3).getReg() != AArch64::XZR;
     72   default:
     73     return false;
     74   }
     75 }
     76 
     77 
     78 //===----------------------------------------------------------------------===//
     79 
     80 namespace {
     81 class AArch64A53Fix835769 : public MachineFunctionPass {
     82   const TargetInstrInfo *TII;
     83 
     84 public:
     85   static char ID;
     86   explicit AArch64A53Fix835769() : MachineFunctionPass(ID) {}
     87 
     88   bool runOnMachineFunction(MachineFunction &F) override;
     89 
     90   const char *getPassName() const override {
     91     return "Workaround A53 erratum 835769 pass";
     92   }
     93 
     94   void getAnalysisUsage(AnalysisUsage &AU) const override {
     95     AU.setPreservesCFG();
     96     MachineFunctionPass::getAnalysisUsage(AU);
     97   }
     98 
     99 private:
    100   bool runOnBasicBlock(MachineBasicBlock &MBB);
    101 };
    102 char AArch64A53Fix835769::ID = 0;
    103 
    104 } // end anonymous namespace
    105 
    106 //===----------------------------------------------------------------------===//
    107 
    108 bool
    109 AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) {
    110   DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n");
    111   bool Changed = false;
    112   TII = F.getSubtarget().getInstrInfo();
    113 
    114   for (auto &MBB : F) {
    115     Changed |= runOnBasicBlock(MBB);
    116   }
    117   return Changed;
    118 }
    119 
    120 // Return the block that was fallen through to get to MBB, if any,
    121 // otherwise nullptr.
    122 static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB,
    123                                              const TargetInstrInfo *TII) {
    124   // Get the previous machine basic block in the function.
    125   MachineFunction::iterator MBBI(MBB);
    126 
    127   // Can't go off top of function.
    128   if (MBBI == MBB->getParent()->begin())
    129     return nullptr;
    130 
    131   MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
    132   SmallVector<MachineOperand, 2> Cond;
    133 
    134   MachineBasicBlock *PrevBB = &*std::prev(MBBI);
    135   for (MachineBasicBlock *S : MBB->predecessors())
    136     if (S == PrevBB && !TII->AnalyzeBranch(*PrevBB, TBB, FBB, Cond) &&
    137         !TBB && !FBB)
    138       return S;
    139 
    140   return nullptr;
    141 }
    142 
    143 // Iterate through fallen through blocks trying to find a previous non-pseudo if
    144 // there is one, otherwise return nullptr. Only look for instructions in
    145 // previous blocks, not the current block, since we only use this to look at
    146 // previous blocks.
    147 static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB,
    148                                       const TargetInstrInfo *TII) {
    149   MachineBasicBlock *FMBB = &MBB;
    150 
    151   // If there is no non-pseudo in the current block, loop back around and try
    152   // the previous block (if there is one).
    153   while ((FMBB = getBBFallenThrough(FMBB, TII))) {
    154     for (MachineInstr &I : make_range(FMBB->rbegin(), FMBB->rend()))
    155       if (!I.isPseudo())
    156         return &I;
    157   }
    158 
    159   // There was no previous non-pseudo in the fallen through blocks
    160   return nullptr;
    161 }
    162 
    163 static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI,
    164                                        const TargetInstrInfo *TII) {
    165   // If we are the first instruction of the block, put the NOP at the end of
    166   // the previous fallthrough block
    167   if (MI == &MBB.front()) {
    168     MachineInstr *I = getLastNonPseudo(MBB, TII);
    169     assert(I && "Expected instruction");
    170     DebugLoc DL = I->getDebugLoc();
    171     BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0);
    172   }
    173   else {
    174     DebugLoc DL = MI->getDebugLoc();
    175     BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0);
    176   }
    177 
    178   ++NumNopsAdded;
    179 }
    180 
    181 bool
    182 AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) {
    183   bool Changed = false;
    184   DEBUG(dbgs() << "Running on MBB: " << MBB << " - scanning instructions...\n");
    185 
    186   // First, scan the basic block, looking for a sequence of 2 instructions
    187   // that match the conditions under which the erratum may trigger.
    188 
    189   // List of terminating instructions in matching sequences
    190   std::vector<MachineInstr*> Sequences;
    191   unsigned Idx = 0;
    192   MachineInstr *PrevInstr = nullptr;
    193 
    194   // Try and find the last non-pseudo instruction in any fallen through blocks,
    195   // if there isn't one, then we use nullptr to represent that.
    196   PrevInstr = getLastNonPseudo(MBB, TII);
    197 
    198   for (auto &MI : MBB) {
    199     MachineInstr *CurrInstr = &MI;
    200     DEBUG(dbgs() << "  Examining: " << MI);
    201     if (PrevInstr) {
    202       DEBUG(dbgs() << "    PrevInstr: " << *PrevInstr
    203                    << "    CurrInstr: " << *CurrInstr
    204                    << "    isFirstInstructionInSequence(PrevInstr): "
    205                    << isFirstInstructionInSequence(PrevInstr) << "\n"
    206                    << "    isSecondInstructionInSequence(CurrInstr): "
    207                    << isSecondInstructionInSequence(CurrInstr) << "\n");
    208       if (isFirstInstructionInSequence(PrevInstr) &&
    209           isSecondInstructionInSequence(CurrInstr)) {
    210         DEBUG(dbgs() << "   ** pattern found at Idx " << Idx << "!\n");
    211         Sequences.push_back(CurrInstr);
    212       }
    213     }
    214     if (!CurrInstr->isPseudo())
    215       PrevInstr = CurrInstr;
    216     ++Idx;
    217   }
    218 
    219   DEBUG(dbgs() << "Scan complete, " << Sequences.size()
    220                << " occurrences of pattern found.\n");
    221 
    222   // Then update the basic block, inserting nops between the detected sequences.
    223   for (auto &MI : Sequences) {
    224     Changed = true;
    225     insertNopBeforeInstruction(MBB, MI, TII);
    226   }
    227 
    228   return Changed;
    229 }
    230 
    231 // Factory function used by AArch64TargetMachine to add the pass to
    232 // the passmanager.
    233 FunctionPass *llvm::createAArch64A53Fix835769() {
    234   return new AArch64A53Fix835769();
    235 }
    236