Home | History | Annotate | Download | only in AArch64 
      1 //===-- AArch64A53Fix835769.cpp -------------------------------------------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 // This pass changes code to work around Cortex-A53 erratum 835769.
     10 // It works around it by inserting a nop instruction in code sequences that
     11 // in some circumstances may trigger the erratum.
     12 // It inserts a nop instruction between a sequence of the following 2 classes
     13 // of instructions:
     14 // instr 1: mem-instr (including loads, stores and prefetches).
     15 // instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers.
     16 //===----------------------------------------------------------------------===//
     17 
     18 #include "AArch64.h"
     19 #include "llvm/ADT/Statistic.h"
     20 #include "llvm/CodeGen/MachineFunction.h"
     21 #include "llvm/CodeGen/MachineFunctionPass.h"
     22 #include "llvm/CodeGen/MachineInstr.h"
     23 #include "llvm/CodeGen/MachineInstrBuilder.h"
     24 #include "llvm/CodeGen/MachineRegisterInfo.h"
     25 #include "llvm/Support/Debug.h"
     26 #include "llvm/Support/raw_ostream.h"
     27 #include "llvm/Target/TargetInstrInfo.h"
     28 
     29 using namespace llvm;
     30 
     31 #define DEBUG_TYPE "aarch64-fix-cortex-a53-835769"
     32 
     33 STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769");
     34 
     35 //===----------------------------------------------------------------------===//
     36 // Helper functions
     37 
     38 // Is the instruction a match for the instruction that comes first in the
     39 // sequence of instructions that can trigger the erratum?
     40 static bool isFirstInstructionInSequence(MachineInstr *MI) {
     41   // Must return true if this instruction is a load, a store or a prefetch.
     42   switch (MI->getOpcode()) {
     43   case AArch64::PRFMl:
     44   case AArch64::PRFMroW:
     45   case AArch64::PRFMroX:
     46   case AArch64::PRFMui:
     47   case AArch64::PRFUMi:
     48     return true;
     49   default:
     50     return MI->mayLoadOrStore();
     51   }
     52 }
     53 
     54 // Is the instruction a match for the instruction that comes second in the
     55 // sequence that can trigger the erratum?
     56 static bool isSecondInstructionInSequence(MachineInstr *MI) {
     57   // Must return true for non-SIMD integer multiply-accumulates, writing
     58   // to a 64-bit register.
     59   switch (MI->getOpcode()) {
     60   // Erratum cannot be triggered when the destination register is 32 bits,
     61   // therefore only include the following.
     62   case AArch64::MSUBXrrr:
     63   case AArch64::MADDXrrr:
     64   case AArch64::SMADDLrrr:
     65   case AArch64::SMSUBLrrr:
     66   case AArch64::UMADDLrrr:
     67   case AArch64::UMSUBLrrr:
     68     // Erratum can only be triggered by multiply-adds, not by regular
     69     // non-accumulating multiplies, i.e. when Ra=XZR='11111'
     70     return MI->getOperand(3).getReg() != AArch64::XZR;
     71   default:
     72     return false;
     73   }
     74 }
     75 
     76 
     77 //===----------------------------------------------------------------------===//
     78 
     79 namespace {
     80 class AArch64A53Fix835769 : public MachineFunctionPass {
     81   const TargetInstrInfo *TII;
     82 
     83 public:
     84   static char ID;
     85   explicit AArch64A53Fix835769() : MachineFunctionPass(ID) {}
     86 
     87   bool runOnMachineFunction(MachineFunction &F) override;
     88 
     89   MachineFunctionProperties getRequiredProperties() const override {
     90     return MachineFunctionProperties().set(
     91         MachineFunctionProperties::Property::AllVRegsAllocated);
     92   }
     93 
     94   const char *getPassName() const override {
     95     return "Workaround A53 erratum 835769 pass";
     96   }
     97 
     98   void getAnalysisUsage(AnalysisUsage &AU) const override {
     99     AU.setPreservesCFG();
    100     MachineFunctionPass::getAnalysisUsage(AU);
    101   }
    102 
    103 private:
    104   bool runOnBasicBlock(MachineBasicBlock &MBB);
    105 };
    106 char AArch64A53Fix835769::ID = 0;
    107 
    108 } // end anonymous namespace
    109 
    110 //===----------------------------------------------------------------------===//
    111 
    112 bool
    113 AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) {
    114   DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n");
    115   bool Changed = false;
    116   TII = F.getSubtarget().getInstrInfo();
    117 
    118   for (auto &MBB : F) {
    119     Changed |= runOnBasicBlock(MBB);
    120   }
    121   return Changed;
    122 }
    123 
    124 // Return the block that was fallen through to get to MBB, if any,
    125 // otherwise nullptr.
    126 static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB,
    127                                              const TargetInstrInfo *TII) {
    128   // Get the previous machine basic block in the function.
    129   MachineFunction::iterator MBBI(MBB);
    130 
    131   // Can't go off top of function.
    132   if (MBBI == MBB->getParent()->begin())
    133     return nullptr;
    134 
    135   MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
    136   SmallVector<MachineOperand, 2> Cond;
    137 
    138   MachineBasicBlock *PrevBB = &*std::prev(MBBI);
    139   for (MachineBasicBlock *S : MBB->predecessors())
    140     if (S == PrevBB && !TII->analyzeBranch(*PrevBB, TBB, FBB, Cond) && !TBB &&
    141         !FBB)
    142       return S;
    143 
    144   return nullptr;
    145 }
    146 
    147 // Iterate through fallen through blocks trying to find a previous non-pseudo if
    148 // there is one, otherwise return nullptr. Only look for instructions in
    149 // previous blocks, not the current block, since we only use this to look at
    150 // previous blocks.
    151 static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB,
    152                                       const TargetInstrInfo *TII) {
    153   MachineBasicBlock *FMBB = &MBB;
    154 
    155   // If there is no non-pseudo in the current block, loop back around and try
    156   // the previous block (if there is one).
    157   while ((FMBB = getBBFallenThrough(FMBB, TII))) {
    158     for (MachineInstr &I : make_range(FMBB->rbegin(), FMBB->rend()))
    159       if (!I.isPseudo())
    160         return &I;
    161   }
    162 
    163   // There was no previous non-pseudo in the fallen through blocks
    164   return nullptr;
    165 }
    166 
    167 static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI,
    168                                        const TargetInstrInfo *TII) {
    169   // If we are the first instruction of the block, put the NOP at the end of
    170   // the previous fallthrough block
    171   if (MI == &MBB.front()) {
    172     MachineInstr *I = getLastNonPseudo(MBB, TII);
    173     assert(I && "Expected instruction");
    174     DebugLoc DL = I->getDebugLoc();
    175     BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0);
    176   }
    177   else {
    178     DebugLoc DL = MI->getDebugLoc();
    179     BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0);
    180   }
    181 
    182   ++NumNopsAdded;
    183 }
    184 
    185 bool
    186 AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) {
    187   bool Changed = false;
    188   DEBUG(dbgs() << "Running on MBB: " << MBB << " - scanning instructions...\n");
    189 
    190   // First, scan the basic block, looking for a sequence of 2 instructions
    191   // that match the conditions under which the erratum may trigger.
    192 
    193   // List of terminating instructions in matching sequences
    194   std::vector<MachineInstr*> Sequences;
    195   unsigned Idx = 0;
    196   MachineInstr *PrevInstr = nullptr;
    197 
    198   // Try and find the last non-pseudo instruction in any fallen through blocks,
    199   // if there isn't one, then we use nullptr to represent that.
    200   PrevInstr = getLastNonPseudo(MBB, TII);
    201 
    202   for (auto &MI : MBB) {
    203     MachineInstr *CurrInstr = &MI;
    204     DEBUG(dbgs() << "  Examining: " << MI);
    205     if (PrevInstr) {
    206       DEBUG(dbgs() << "    PrevInstr: " << *PrevInstr
    207                    << "    CurrInstr: " << *CurrInstr
    208                    << "    isFirstInstructionInSequence(PrevInstr): "
    209                    << isFirstInstructionInSequence(PrevInstr) << "\n"
    210                    << "    isSecondInstructionInSequence(CurrInstr): "
    211                    << isSecondInstructionInSequence(CurrInstr) << "\n");
    212       if (isFirstInstructionInSequence(PrevInstr) &&
    213           isSecondInstructionInSequence(CurrInstr)) {
    214         DEBUG(dbgs() << "   ** pattern found at Idx " << Idx << "!\n");
    215         Sequences.push_back(CurrInstr);
    216       }
    217     }
    218     if (!CurrInstr->isPseudo())
    219       PrevInstr = CurrInstr;
    220     ++Idx;
    221   }
    222 
    223   DEBUG(dbgs() << "Scan complete, " << Sequences.size()
    224                << " occurrences of pattern found.\n");
    225 
    226   // Then update the basic block, inserting nops between the detected sequences.
    227   for (auto &MI : Sequences) {
    228     Changed = true;
    229     insertNopBeforeInstruction(MBB, MI, TII);
    230   }
    231 
    232   return Changed;
    233 }
    234 
    235 // Factory function used by AArch64TargetMachine to add the pass to
    236 // the passmanager.
    237 FunctionPass *llvm::createAArch64A53Fix835769() {
    238   return new AArch64A53Fix835769();
    239 }
    240