Home | History | Annotate | Download | only in src 
      1 //===- subzero/src/IceInstARM32.cpp - ARM32 instruction implementation ----===//
      2 //
      3 //                        The Subzero Code Generator
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 ///
     10 /// \file
     11 /// \brief Implements the InstARM32 and OperandARM32 classes, primarily the
     12 /// constructors and the dump()/emit() methods.
     13 ///
     14 //===----------------------------------------------------------------------===//
     15 
     16 #include "IceInstARM32.h"
     17 
     18 #include "IceAssemblerARM32.h"
     19 #include "IceCfg.h"
     20 #include "IceCfgNode.h"
     21 #include "IceInst.h"
     22 #include "IceOperand.h"
     23 #include "IceTargetLoweringARM32.h"
     24 
     25 namespace Ice {
     26 namespace ARM32 {
     27 
     28 namespace {
     29 
     30 using Register = RegARM32::AllRegisters;
     31 
     32 // maximum number of registers allowed in vpush/vpop.
     33 static constexpr SizeT VpushVpopMaxConsecRegs = 16;
     34 
     35 const struct TypeARM32Attributes_ {
     36   const char *WidthString;     // b, h, <blank>, or d
     37   const char *FpWidthString;   // i8, i16, i32, f32, f64
     38   const char *SVecWidthString; // s8, s16, s32, f32
     39   const char *UVecWidthString; // u8, u16, u32, f32
     40   int8_t SExtAddrOffsetBits;
     41   int8_t ZExtAddrOffsetBits;
     42 } TypeARM32Attributes[] = {
     43 #define X(tag, elementty, int_width, fp_width, uvec_width, svec_width, sbits,  \
     44           ubits, rraddr, shaddr)                                               \
     45   { int_width, fp_width, svec_width, uvec_width, sbits, ubits }                \
     46   ,
     47     ICETYPEARM32_TABLE
     48 #undef X
     49 };
     50 
     51 const struct InstARM32ShiftAttributes_ {
     52   const char *EmitString;
     53 } InstARM32ShiftAttributes[] = {
     54 #define X(tag, emit)                                                           \
     55   { emit }                                                                     \
     56   ,
     57     ICEINSTARM32SHIFT_TABLE
     58 #undef X
     59 };
     60 
     61 const struct InstARM32CondAttributes_ {
     62   CondARM32::Cond Opposite;
     63   const char *EmitString;
     64 } InstARM32CondAttributes[] = {
     65 #define X(tag, encode, opp, emit)                                              \
     66   { CondARM32::opp, emit }                                                     \
     67   ,
     68     ICEINSTARM32COND_TABLE
     69 #undef X
     70 };
     71 
     72 size_t getVecElmtBitsize(Type Ty) {
     73   return typeWidthInBytes(typeElementType(Ty)) * CHAR_BIT;
     74 }
     75 
     76 const char *getWidthString(Type Ty) {
     77   return TypeARM32Attributes[Ty].WidthString;
     78 }
     79 
     80 const char *getFpWidthString(Type Ty) {
     81   return TypeARM32Attributes[Ty].FpWidthString;
     82 }
     83 
     84 const char *getSVecWidthString(Type Ty) {
     85   return TypeARM32Attributes[Ty].SVecWidthString;
     86 }
     87 
     88 const char *getUVecWidthString(Type Ty) {
     89   return TypeARM32Attributes[Ty].UVecWidthString;
     90 }
     91 
     92 const char *getVWidthString(Type Ty, InstARM32::FPSign SignType) {
     93   switch (SignType) {
     94   case InstARM32::FS_None:
     95     return getFpWidthString(Ty);
     96   case InstARM32::FS_Signed:
     97     return getSVecWidthString(Ty);
     98   case InstARM32::FS_Unsigned:
     99     return getUVecWidthString(Ty);
    100   }
    101   llvm_unreachable("Invalid Sign Type.");
    102   return getFpWidthString(Ty);
    103 }
    104 
    105 } // end of anonymous namespace
    106 
    107 const char *InstARM32Pred::predString(CondARM32::Cond Pred) {
    108   return InstARM32CondAttributes[Pred].EmitString;
    109 }
    110 
    111 void InstARM32Pred::dumpOpcodePred(Ostream &Str, const char *Opcode,
    112                                    Type Ty) const {
    113   Str << Opcode << getPredicate() << "." << Ty;
    114 }
    115 
    116 CondARM32::Cond InstARM32::getOppositeCondition(CondARM32::Cond Cond) {
    117   return InstARM32CondAttributes[Cond].Opposite;
    118 }
    119 
    120 void InstARM32::startNextInst(const Cfg *Func) const {
    121   if (auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>())
    122     Asm->incEmitTextSize(InstSize);
    123 }
    124 
    125 void InstARM32::emitUsingTextFixup(const Cfg *Func) const {
    126   if (!BuildDefs::dump())
    127     return;
    128   GlobalContext *Ctx = Func->getContext();
    129   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    130   if (getFlags().getDisableHybridAssembly() &&
    131       getFlags().getSkipUnimplemented()) {
    132     Asm->trap();
    133     Asm->resetNeedsTextFixup();
    134     return;
    135   }
    136   std::string Buffer;
    137   llvm::raw_string_ostream StrBuf(Buffer);
    138   OstreamLocker L(Ctx);
    139   Ostream &OldStr = Ctx->getStrEmit();
    140   Ctx->setStrEmit(StrBuf);
    141   // Start counting instructions here, so that emit() methods don't
    142   // need to call this for the first instruction.
    143   Asm->resetEmitTextSize();
    144   Asm->incEmitTextSize(InstSize);
    145   emit(Func);
    146   Ctx->setStrEmit(OldStr);
    147   if (getFlags().getDisableHybridAssembly()) {
    148     if (getFlags().getSkipUnimplemented()) {
    149       Asm->trap();
    150     } else {
    151       llvm::errs() << "Can't assemble: " << StrBuf.str() << "\n";
    152       UnimplementedError(getFlags());
    153     }
    154     Asm->resetNeedsTextFixup();
    155     return;
    156   }
    157   Asm->emitTextInst(StrBuf.str(), Asm->getEmitTextSize());
    158 }
    159 
    160 void InstARM32::emitIAS(const Cfg *Func) const { emitUsingTextFixup(Func); }
    161 
    162 void InstARM32Pred::emitUnaryopGPR(const char *Opcode,
    163                                    const InstARM32Pred *Instr, const Cfg *Func,
    164                                    bool NeedsWidthSuffix) {
    165   Ostream &Str = Func->getContext()->getStrEmit();
    166   assert(Instr->getSrcSize() == 1);
    167   Type SrcTy = Instr->getSrc(0)->getType();
    168   Str << "\t" << Opcode;
    169   if (NeedsWidthSuffix)
    170     Str << getWidthString(SrcTy);
    171   Str << Instr->getPredicate() << "\t";
    172   Instr->getDest()->emit(Func);
    173   Str << ", ";
    174   Instr->getSrc(0)->emit(Func);
    175 }
    176 
    177 void InstARM32Pred::emitUnaryopFP(const char *Opcode, FPSign Sign,
    178                                   const InstARM32Pred *Instr, const Cfg *Func) {
    179   Ostream &Str = Func->getContext()->getStrEmit();
    180   assert(Instr->getSrcSize() == 1);
    181   Type SrcTy = Instr->getSrc(0)->getType();
    182   Str << "\t" << Opcode << Instr->getPredicate();
    183   switch (Sign) {
    184   case FS_None:
    185     Str << getFpWidthString(SrcTy);
    186     break;
    187   case FS_Signed:
    188     Str << getSVecWidthString(SrcTy);
    189     break;
    190   case FS_Unsigned:
    191     Str << getUVecWidthString(SrcTy);
    192     break;
    193   }
    194   Str << "\t";
    195   Instr->getDest()->emit(Func);
    196   Str << ", ";
    197   Instr->getSrc(0)->emit(Func);
    198 }
    199 
    200 void InstARM32Pred::emitTwoAddr(const char *Opcode, const InstARM32Pred *Instr,
    201                                 const Cfg *Func) {
    202   if (!BuildDefs::dump())
    203     return;
    204   Ostream &Str = Func->getContext()->getStrEmit();
    205   assert(Instr->getSrcSize() == 2);
    206   Variable *Dest = Instr->getDest();
    207   assert(Dest == Instr->getSrc(0));
    208   Str << "\t" << Opcode << Instr->getPredicate() << "\t";
    209   Dest->emit(Func);
    210   Str << ", ";
    211   Instr->getSrc(1)->emit(Func);
    212 }
    213 
    214 void InstARM32Pred::emitThreeAddr(const char *Opcode,
    215                                   const InstARM32Pred *Instr, const Cfg *Func,
    216                                   bool SetFlags) {
    217   if (!BuildDefs::dump())
    218     return;
    219   Ostream &Str = Func->getContext()->getStrEmit();
    220   assert(Instr->getSrcSize() == 2);
    221   Str << "\t" << Opcode << (SetFlags ? "s" : "") << Instr->getPredicate()
    222       << "\t";
    223   Instr->getDest()->emit(Func);
    224   Str << ", ";
    225   Instr->getSrc(0)->emit(Func);
    226   Str << ", ";
    227   Instr->getSrc(1)->emit(Func);
    228 }
    229 
    230 void InstARM32::emitThreeAddrFP(const char *Opcode, FPSign SignType,
    231                                 const InstARM32 *Instr, const Cfg *Func,
    232                                 Type OpType) {
    233   if (!BuildDefs::dump())
    234     return;
    235   Ostream &Str = Func->getContext()->getStrEmit();
    236   assert(Instr->getSrcSize() == 2);
    237   Str << "\t" << Opcode << getVWidthString(OpType, SignType) << "\t";
    238   Instr->getDest()->emit(Func);
    239   Str << ", ";
    240   Instr->getSrc(0)->emit(Func);
    241   Str << ", ";
    242   Instr->getSrc(1)->emit(Func);
    243 }
    244 
    245 void InstARM32::emitFourAddrFP(const char *Opcode, FPSign SignType,
    246                                const InstARM32 *Instr, const Cfg *Func) {
    247   if (!BuildDefs::dump())
    248     return;
    249   Ostream &Str = Func->getContext()->getStrEmit();
    250   assert(Instr->getSrcSize() == 3);
    251   assert(Instr->getSrc(0) == Instr->getDest());
    252   Str << "\t" << Opcode
    253       << getVWidthString(Instr->getDest()->getType(), SignType) << "\t";
    254   Instr->getDest()->emit(Func);
    255   Str << ", ";
    256   Instr->getSrc(1)->emit(Func);
    257   Str << ", ";
    258   Instr->getSrc(2)->emit(Func);
    259 }
    260 
    261 void InstARM32Pred::emitFourAddr(const char *Opcode, const InstARM32Pred *Instr,
    262                                  const Cfg *Func) {
    263   if (!BuildDefs::dump())
    264     return;
    265   Ostream &Str = Func->getContext()->getStrEmit();
    266   assert(Instr->getSrcSize() == 3);
    267   Str << "\t" << Opcode << Instr->getPredicate() << "\t";
    268   Instr->getDest()->emit(Func);
    269   Str << ", ";
    270   Instr->getSrc(0)->emit(Func);
    271   Str << ", ";
    272   Instr->getSrc(1)->emit(Func);
    273   Str << ", ";
    274   Instr->getSrc(2)->emit(Func);
    275 }
    276 
    277 template <InstARM32::InstKindARM32 K>
    278 void InstARM32FourAddrGPR<K>::emitIAS(const Cfg *Func) const {
    279   emitUsingTextFixup(Func);
    280 }
    281 
    282 template <InstARM32::InstKindARM32 K>
    283 void InstARM32FourAddrFP<K>::emitIAS(const Cfg *Func) const {
    284   emitUsingTextFixup(Func);
    285 }
    286 
    287 template <InstARM32::InstKindARM32 K>
    288 void InstARM32ThreeAddrFP<K>::emitIAS(const Cfg *Func) const {
    289   emitUsingTextFixup(Func);
    290 }
    291 
    292 template <InstARM32::InstKindARM32 K>
    293 void InstARM32ThreeAddrSignAwareFP<K>::emitIAS(const Cfg *Func) const {
    294   InstARM32::emitUsingTextFixup(Func);
    295 }
    296 
    297 template <> void InstARM32Mla::emitIAS(const Cfg *Func) const {
    298   assert(getSrcSize() == 3);
    299   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    300   Asm->mla(getDest(), getSrc(0), getSrc(1), getSrc(2), getPredicate());
    301   if (Asm->needsTextFixup())
    302     emitUsingTextFixup(Func);
    303 }
    304 
    305 template <> void InstARM32Mls::emitIAS(const Cfg *Func) const {
    306   assert(getSrcSize() == 3);
    307   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    308   Asm->mls(getDest(), getSrc(0), getSrc(1), getSrc(2), getPredicate());
    309   if (Asm->needsTextFixup())
    310     emitUsingTextFixup(Func);
    311 }
    312 
    313 void InstARM32Pred::emitCmpLike(const char *Opcode, const InstARM32Pred *Instr,
    314                                 const Cfg *Func) {
    315   if (!BuildDefs::dump())
    316     return;
    317   Ostream &Str = Func->getContext()->getStrEmit();
    318   assert(Instr->getSrcSize() == 2);
    319   Str << "\t" << Opcode << Instr->getPredicate() << "\t";
    320   Instr->getSrc(0)->emit(Func);
    321   Str << ", ";
    322   Instr->getSrc(1)->emit(Func);
    323 }
    324 
    325 OperandARM32Mem::OperandARM32Mem(Cfg * /* Func */, Type Ty, Variable *Base,
    326                                  ConstantInteger32 *ImmOffset, AddrMode Mode)
    327     : OperandARM32(kMem, Ty), Base(Base), ImmOffset(ImmOffset), Index(nullptr),
    328       ShiftOp(kNoShift), ShiftAmt(0), Mode(Mode) {
    329   // The Neg modes are only needed for Reg +/- Reg.
    330   assert(!isNegAddrMode());
    331   NumVars = 1;
    332   Vars = &this->Base;
    333 }
    334 
    335 OperandARM32Mem::OperandARM32Mem(Cfg *Func, Type Ty, Variable *Base,
    336                                  Variable *Index, ShiftKind ShiftOp,
    337                                  uint16_t ShiftAmt, AddrMode Mode)
    338     : OperandARM32(kMem, Ty), Base(Base), ImmOffset(0), Index(Index),
    339       ShiftOp(ShiftOp), ShiftAmt(ShiftAmt), Mode(Mode) {
    340   if (Index->isRematerializable()) {
    341     llvm::report_fatal_error("Rematerializable Index Register is not allowed.");
    342   }
    343   NumVars = 2;
    344   Vars = Func->allocateArrayOf<Variable *>(2);
    345   Vars[0] = Base;
    346   Vars[1] = Index;
    347 }
    348 
    349 OperandARM32ShAmtImm::OperandARM32ShAmtImm(ConstantInteger32 *SA)
    350     : OperandARM32(kShAmtImm, IceType_i8), ShAmt(SA) {}
    351 
    352 bool OperandARM32Mem::canHoldOffset(Type Ty, bool SignExt, int32_t Offset) {
    353   int32_t Bits = SignExt ? TypeARM32Attributes[Ty].SExtAddrOffsetBits
    354                          : TypeARM32Attributes[Ty].ZExtAddrOffsetBits;
    355   if (Bits == 0)
    356     return Offset == 0;
    357   // Note that encodings for offsets are sign-magnitude for ARM, so we check
    358   // with IsAbsoluteUint().
    359   // Scalar fp, and vector types require an offset that is aligned to a multiple
    360   // of 4.
    361   if (isScalarFloatingType(Ty) || isVectorType(Ty))
    362     return Utils::IsAligned(Offset, 4) && Utils::IsAbsoluteUint(Bits, Offset);
    363   return Utils::IsAbsoluteUint(Bits, Offset);
    364 }
    365 
    366 OperandARM32FlexImm::OperandARM32FlexImm(Cfg * /* Func */, Type Ty,
    367                                          uint32_t Imm, uint32_t RotateAmt)
    368     : OperandARM32Flex(kFlexImm, Ty), Imm(Imm), RotateAmt(RotateAmt) {
    369   NumVars = 0;
    370   Vars = nullptr;
    371 }
    372 
    373 bool OperandARM32FlexImm::canHoldImm(uint32_t Immediate, uint32_t *RotateAmt,
    374                                      uint32_t *Immed_8) {
    375   // Avoid the more expensive test for frequent small immediate values.
    376   if (Immediate <= 0xFF) {
    377     *RotateAmt = 0;
    378     *Immed_8 = Immediate;
    379     return true;
    380   }
    381   // Note that immediate must be unsigned for the test to work correctly.
    382   for (int Rot = 1; Rot < 16; Rot++) {
    383     uint32_t Imm8 = Utils::rotateLeft32(Immediate, 2 * Rot);
    384     if (Imm8 <= 0xFF) {
    385       *RotateAmt = Rot;
    386       *Immed_8 = Imm8;
    387       return true;
    388     }
    389   }
    390   return false;
    391 }
    392 
    393 OperandARM32FlexFpImm::OperandARM32FlexFpImm(Cfg * /*Func*/, Type Ty,
    394                                              uint32_t ModifiedImm)
    395     : OperandARM32Flex(kFlexFpImm, Ty), ModifiedImm(ModifiedImm) {}
    396 
    397 bool OperandARM32FlexFpImm::canHoldImm(const Operand *C,
    398                                        uint32_t *ModifiedImm) {
    399   switch (C->getType()) {
    400   default:
    401     llvm::report_fatal_error("Unhandled fp constant type.");
    402   case IceType_f32: {
    403     // We violate llvm naming conventions a bit here so that the constants are
    404     // named after the bit fields they represent. See "A7.5.1 Operation of
    405     // modified immediate constants, Floating-point" in the ARM ARM.
    406     static constexpr uint32_t a = 0x80000000u;
    407     static constexpr uint32_t B = 0x40000000;
    408     static constexpr uint32_t bbbbb = 0x3E000000;
    409     static constexpr uint32_t cdefgh = 0x01F80000;
    410     static constexpr uint32_t AllowedBits = a | B | bbbbb | cdefgh;
    411     static_assert(AllowedBits == 0xFFF80000u,
    412                   "Invalid mask for f32 modified immediates.");
    413     const float F32 = llvm::cast<const ConstantFloat>(C)->getValue();
    414     const uint32_t I32 = Utils::bitCopy<uint32_t>(F32);
    415     if (I32 & ~AllowedBits) {
    416       // constant has disallowed bits.
    417       return false;
    418     }
    419 
    420     if ((I32 & bbbbb) != bbbbb && (I32 & bbbbb)) {
    421       // not all bbbbb bits are 0 or 1.
    422       return false;
    423     }
    424 
    425     if (((I32 & B) != 0) == ((I32 & bbbbb) != 0)) {
    426       // B ^ b = 0;
    427       return false;
    428     }
    429 
    430     *ModifiedImm = ((I32 & a) ? 0x80 : 0x00) | ((I32 & bbbbb) ? 0x40 : 0x00) |
    431                    ((I32 & cdefgh) >> 19);
    432     return true;
    433   }
    434   case IceType_f64: {
    435     static constexpr uint32_t a = 0x80000000u;
    436     static constexpr uint32_t B = 0x40000000;
    437     static constexpr uint32_t bbbbbbbb = 0x3FC00000;
    438     static constexpr uint32_t cdefgh = 0x003F0000;
    439     static constexpr uint32_t AllowedBits = a | B | bbbbbbbb | cdefgh;
    440     static_assert(AllowedBits == 0xFFFF0000u,
    441                   "Invalid mask for f64 modified immediates.");
    442     const double F64 = llvm::cast<const ConstantDouble>(C)->getValue();
    443     const uint64_t I64 = Utils::bitCopy<uint64_t>(F64);
    444     if (I64 & 0xFFFFFFFFu) {
    445       // constant has disallowed bits.
    446       return false;
    447     }
    448     const uint32_t I32 = I64 >> 32;
    449 
    450     if (I32 & ~AllowedBits) {
    451       // constant has disallowed bits.
    452       return false;
    453     }
    454 
    455     if ((I32 & bbbbbbbb) != bbbbbbbb && (I32 & bbbbbbbb)) {
    456       // not all bbbbb bits are 0 or 1.
    457       return false;
    458     }
    459 
    460     if (((I32 & B) != 0) == ((I32 & bbbbbbbb) != 0)) {
    461       // B ^ b = 0;
    462       return false;
    463     }
    464 
    465     *ModifiedImm = ((I32 & a) ? 0x80 : 0x00) |
    466                    ((I32 & bbbbbbbb) ? 0x40 : 0x00) | ((I32 & cdefgh) >> 16);
    467     return true;
    468   }
    469   }
    470 }
    471 
    472 OperandARM32FlexFpZero::OperandARM32FlexFpZero(Cfg * /*Func*/, Type Ty)
    473     : OperandARM32Flex(kFlexFpZero, Ty) {}
    474 
    475 OperandARM32FlexReg::OperandARM32FlexReg(Cfg *Func, Type Ty, Variable *Reg,
    476                                          ShiftKind ShiftOp, Operand *ShiftAmt)
    477     : OperandARM32Flex(kFlexReg, Ty), Reg(Reg), ShiftOp(ShiftOp),
    478       ShiftAmt(ShiftAmt) {
    479   NumVars = 1;
    480   auto *ShiftVar = llvm::dyn_cast_or_null<Variable>(ShiftAmt);
    481   if (ShiftVar)
    482     ++NumVars;
    483   Vars = Func->allocateArrayOf<Variable *>(NumVars);
    484   Vars[0] = Reg;
    485   if (ShiftVar)
    486     Vars[1] = ShiftVar;
    487 }
    488 
    489 InstARM32Br::InstARM32Br(Cfg *Func, const CfgNode *TargetTrue,
    490                          const CfgNode *TargetFalse,
    491                          const InstARM32Label *Label, CondARM32::Cond Pred)
    492     : InstARM32Pred(Func, InstARM32::Br, 0, nullptr, Pred),
    493       TargetTrue(TargetTrue), TargetFalse(TargetFalse), Label(Label) {}
    494 
    495 bool InstARM32Br::optimizeBranch(const CfgNode *NextNode) {
    496   // If there is no next block, then there can be no fallthrough to optimize.
    497   if (NextNode == nullptr)
    498     return false;
    499   // Intra-block conditional branches can't be optimized.
    500   if (Label)
    501     return false;
    502   // If there is no fallthrough node, such as a non-default case label for a
    503   // switch instruction, then there is no opportunity to optimize.
    504   if (getTargetFalse() == nullptr)
    505     return false;
    506 
    507   // Unconditional branch to the next node can be removed.
    508   if (isUnconditionalBranch() && getTargetFalse() == NextNode) {
    509     assert(getTargetTrue() == nullptr);
    510     setDeleted();
    511     return true;
    512   }
    513   // If the fallthrough is to the next node, set fallthrough to nullptr to
    514   // indicate.
    515   if (getTargetFalse() == NextNode) {
    516     TargetFalse = nullptr;
    517     return true;
    518   }
    519   // If TargetTrue is the next node, and TargetFalse is not nullptr (which was
    520   // already tested above), then invert the branch condition, swap the targets,
    521   // and set new fallthrough to nullptr.
    522   if (getTargetTrue() == NextNode) {
    523     assert(Predicate != CondARM32::AL);
    524     setPredicate(getOppositeCondition(getPredicate()));
    525     TargetTrue = getTargetFalse();
    526     TargetFalse = nullptr;
    527     return true;
    528   }
    529   return false;
    530 }
    531 
    532 bool InstARM32Br::repointEdges(CfgNode *OldNode, CfgNode *NewNode) {
    533   bool Found = false;
    534   if (TargetFalse == OldNode) {
    535     TargetFalse = NewNode;
    536     Found = true;
    537   }
    538   if (TargetTrue == OldNode) {
    539     TargetTrue = NewNode;
    540     Found = true;
    541   }
    542   return Found;
    543 }
    544 
    545 template <InstARM32::InstKindARM32 K>
    546 void InstARM32ThreeAddrGPR<K>::emitIAS(const Cfg *Func) const {
    547   emitUsingTextFixup(Func);
    548 }
    549 
    550 template <> void InstARM32Adc::emitIAS(const Cfg *Func) const {
    551   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    552   Asm->adc(getDest(), getSrc(0), getSrc(1), SetFlags, getPredicate());
    553   if (Asm->needsTextFixup())
    554     emitUsingTextFixup(Func);
    555 }
    556 
    557 template <> void InstARM32Add::emitIAS(const Cfg *Func) const {
    558   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    559   Asm->add(getDest(), getSrc(0), getSrc(1), SetFlags, getPredicate());
    560   assert(!Asm->needsTextFixup());
    561 }
    562 
    563 template <> void InstARM32And::emitIAS(const Cfg *Func) const {
    564   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    565   Asm->and_(getDest(), getSrc(0), getSrc(1), SetFlags, getPredicate());
    566   if (Asm->needsTextFixup())
    567     emitUsingTextFixup(Func);
    568 }
    569 
    570 template <> void InstARM32Bic::emitIAS(const Cfg *Func) const {
    571   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    572   Asm->bic(getDest(), getSrc(0), getSrc(1), SetFlags, getPredicate());
    573   if (Asm->needsTextFixup())
    574     emitUsingTextFixup(Func);
    575 }
    576 
    577 template <> void InstARM32Eor::emitIAS(const Cfg *Func) const {
    578   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    579   Asm->eor(getDest(), getSrc(0), getSrc(1), SetFlags, getPredicate());
    580   if (Asm->needsTextFixup())
    581     emitUsingTextFixup(Func);
    582 }
    583 
    584 template <> void InstARM32Asr::emitIAS(const Cfg *Func) const {
    585   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    586   Asm->asr(getDest(), getSrc(0), getSrc(1), SetFlags, getPredicate());
    587   if (Asm->needsTextFixup())
    588     emitUsingTextFixup(Func);
    589 }
    590 
    591 template <> void InstARM32Lsl::emitIAS(const Cfg *Func) const {
    592   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    593   Asm->lsl(getDest(), getSrc(0), getSrc(1), SetFlags, getPredicate());
    594   if (Asm->needsTextFixup())
    595     emitUsingTextFixup(Func);
    596 }
    597 
    598 template <> void InstARM32Lsr::emitIAS(const Cfg *Func) const {
    599   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    600   Asm->lsr(getDest(), getSrc(0), getSrc(1), SetFlags, getPredicate());
    601   if (Asm->needsTextFixup())
    602     emitUsingTextFixup(Func);
    603 }
    604 
    605 template <> void InstARM32Orr::emitIAS(const Cfg *Func) const {
    606   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    607   Asm->orr(getDest(), getSrc(0), getSrc(1), SetFlags, getPredicate());
    608   if (Asm->needsTextFixup())
    609     emitUsingTextFixup(Func);
    610 }
    611 
    612 template <> void InstARM32Mul::emitIAS(const Cfg *Func) const {
    613   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    614   Asm->mul(getDest(), getSrc(0), getSrc(1), SetFlags, getPredicate());
    615   if (Asm->needsTextFixup())
    616     emitUsingTextFixup(Func);
    617 }
    618 
    619 template <> void InstARM32Rsb::emitIAS(const Cfg *Func) const {
    620   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    621   Asm->rsb(getDest(), getSrc(0), getSrc(1), SetFlags, getPredicate());
    622   if (Asm->needsTextFixup())
    623     emitUsingTextFixup(Func);
    624 }
    625 
    626 template <> void InstARM32Rsc::emitIAS(const Cfg *Func) const {
    627   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    628   Asm->rsc(getDest(), getSrc(0), getSrc(1), SetFlags, getPredicate());
    629   if (Asm->needsTextFixup())
    630     emitUsingTextFixup(Func);
    631 }
    632 
    633 template <> void InstARM32Sbc::emitIAS(const Cfg *Func) const {
    634   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    635   Asm->sbc(getDest(), getSrc(0), getSrc(1), SetFlags, getPredicate());
    636   if (Asm->needsTextFixup())
    637     emitUsingTextFixup(Func);
    638 }
    639 
    640 template <> void InstARM32Sdiv::emitIAS(const Cfg *Func) const {
    641   assert(!SetFlags);
    642   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    643   Asm->sdiv(getDest(), getSrc(0), getSrc(1), getPredicate());
    644   if (Asm->needsTextFixup())
    645     emitUsingTextFixup(Func);
    646 }
    647 
    648 template <> void InstARM32Sub::emitIAS(const Cfg *Func) const {
    649   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    650   Asm->sub(getDest(), getSrc(0), getSrc(1), SetFlags, getPredicate());
    651   if (Asm->needsTextFixup())
    652     emitUsingTextFixup(Func);
    653 }
    654 
    655 template <> void InstARM32Udiv::emitIAS(const Cfg *Func) const {
    656   assert(!SetFlags);
    657   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    658   Asm->udiv(getDest(), getSrc(0), getSrc(1), getPredicate());
    659   if (Asm->needsTextFixup())
    660     emitUsingTextFixup(Func);
    661 }
    662 
    663 template <> void InstARM32Vadd::emitIAS(const Cfg *Func) const {
    664   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    665   const Variable *Dest = getDest();
    666   Type DestTy = Dest->getType();
    667   switch (DestTy) {
    668   default:
    669     llvm::report_fatal_error("Vadd not defined on type " +
    670                              typeStdString(DestTy));
    671   case IceType_v16i8:
    672   case IceType_v8i16:
    673   case IceType_v4i32:
    674     Asm->vaddqi(typeElementType(DestTy), Dest, getSrc(0), getSrc(1));
    675     break;
    676   case IceType_v4f32:
    677     Asm->vaddqf(Dest, getSrc(0), getSrc(1));
    678     break;
    679   case IceType_f32:
    680     Asm->vadds(Dest, getSrc(0), getSrc(1), CondARM32::AL);
    681     break;
    682   case IceType_f64:
    683     Asm->vaddd(Dest, getSrc(0), getSrc(1), CondARM32::AL);
    684     break;
    685   }
    686   assert(!Asm->needsTextFixup());
    687 }
    688 
    689 template <> void InstARM32Vand::emitIAS(const Cfg *Func) const {
    690   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    691   const Variable *Dest = getDest();
    692   switch (Dest->getType()) {
    693   default:
    694     llvm::report_fatal_error("Vand not defined on type " +
    695                              typeStdString(Dest->getType()));
    696   case IceType_v4i1:
    697   case IceType_v8i1:
    698   case IceType_v16i1:
    699   case IceType_v16i8:
    700   case IceType_v8i16:
    701   case IceType_v4i32:
    702     Asm->vandq(Dest, getSrc(0), getSrc(1));
    703   }
    704   assert(!Asm->needsTextFixup());
    705 }
    706 
    707 template <> void InstARM32Vceq::emitIAS(const Cfg *Func) const {
    708   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    709   const Variable *Dest = getDest();
    710   const Type SrcTy = getSrc(0)->getType();
    711   switch (SrcTy) {
    712   default:
    713     llvm::report_fatal_error("Vceq not defined on type " +
    714                              typeStdString(SrcTy));
    715   case IceType_v4i1:
    716   case IceType_v8i1:
    717   case IceType_v16i1:
    718   case IceType_v16i8:
    719   case IceType_v8i16:
    720   case IceType_v4i32:
    721     Asm->vceqqi(typeElementType(SrcTy), Dest, getSrc(0), getSrc(1));
    722     break;
    723   case IceType_v4f32:
    724     Asm->vceqqs(Dest, getSrc(0), getSrc(1));
    725     break;
    726   }
    727   assert(!Asm->needsTextFixup());
    728 }
    729 
    730 template <> void InstARM32Vcge::emitIAS(const Cfg *Func) const {
    731   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    732   const Variable *Dest = getDest();
    733   const Type SrcTy = getSrc(0)->getType();
    734   switch (SrcTy) {
    735   default:
    736     llvm::report_fatal_error("Vcge not defined on type " +
    737                              typeStdString(Dest->getType()));
    738   case IceType_v4i1:
    739   case IceType_v8i1:
    740   case IceType_v16i1:
    741   case IceType_v16i8:
    742   case IceType_v8i16:
    743   case IceType_v4i32: {
    744     const Type ElmtTy = typeElementType(SrcTy);
    745     assert(Sign != InstARM32::FS_None);
    746     switch (Sign) {
    747     case InstARM32::FS_None: // defaults to unsigned.
    748       llvm_unreachable("Sign should not be FS_None.");
    749     case InstARM32::FS_Unsigned:
    750       Asm->vcugeqi(ElmtTy, Dest, getSrc(0), getSrc(1));
    751       break;
    752     case InstARM32::FS_Signed:
    753       Asm->vcgeqi(ElmtTy, Dest, getSrc(0), getSrc(1));
    754       break;
    755     }
    756   } break;
    757   case IceType_v4f32:
    758     Asm->vcgeqs(Dest, getSrc(0), getSrc(1));
    759     break;
    760   }
    761 }
    762 
    763 template <> void InstARM32Vcgt::emitIAS(const Cfg *Func) const {
    764   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    765   const Variable *Dest = getDest();
    766   const Type SrcTy = getSrc(0)->getType();
    767   switch (SrcTy) {
    768   default:
    769     llvm::report_fatal_error("Vcgt not defined on type " +
    770                              typeStdString(Dest->getType()));
    771   case IceType_v4i1:
    772   case IceType_v8i1:
    773   case IceType_v16i1:
    774   case IceType_v16i8:
    775   case IceType_v8i16:
    776   case IceType_v4i32: {
    777     const Type ElmtTy = typeElementType(SrcTy);
    778     assert(Sign != InstARM32::FS_None);
    779     switch (Sign) {
    780     case InstARM32::FS_None: // defaults to unsigned.
    781       llvm_unreachable("Sign should not be FS_None.");
    782     case InstARM32::FS_Unsigned:
    783       Asm->vcugtqi(ElmtTy, Dest, getSrc(0), getSrc(1));
    784       break;
    785     case InstARM32::FS_Signed:
    786       Asm->vcgtqi(ElmtTy, Dest, getSrc(0), getSrc(1));
    787       break;
    788     }
    789   } break;
    790   case IceType_v4f32:
    791     Asm->vcgtqs(Dest, getSrc(0), getSrc(1));
    792     break;
    793   }
    794 }
    795 
    796 template <> void InstARM32Vbsl::emitIAS(const Cfg *Func) const {
    797   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    798   const Variable *Dest = getDest();
    799   switch (Dest->getType()) {
    800   default:
    801     llvm::report_fatal_error("Vbsl not defined on type " +
    802                              typeStdString(Dest->getType()));
    803   case IceType_v4i1:
    804   case IceType_v8i1:
    805   case IceType_v16i1:
    806   case IceType_v16i8:
    807   case IceType_v8i16:
    808   case IceType_v4i32:
    809     Asm->vbslq(Dest, getSrc(0), getSrc(1));
    810   }
    811   assert(!Asm->needsTextFixup());
    812 }
    813 
    814 template <> void InstARM32Vdiv::emitIAS(const Cfg *Func) const {
    815   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    816   const Variable *Dest = getDest();
    817   switch (Dest->getType()) {
    818   default:
    819     // TODO(kschimpf) Figure if more cases are needed.
    820     Asm->setNeedsTextFixup();
    821     break;
    822   case IceType_f32:
    823     Asm->vdivs(getDest(), getSrc(0), getSrc(1), CondARM32::AL);
    824     break;
    825   case IceType_f64:
    826     Asm->vdivd(getDest(), getSrc(0), getSrc(1), CondARM32::AL);
    827     break;
    828   }
    829   assert(!Asm->needsTextFixup());
    830 }
    831 
    832 template <> void InstARM32Veor::emitIAS(const Cfg *Func) const {
    833   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    834   const Variable *Dest = getDest();
    835   if (isVectorType(Dest->getType())) {
    836     Asm->veorq(Dest, getSrc(0), getSrc(1));
    837     assert(!Asm->needsTextFixup());
    838     return;
    839   }
    840   assert(Dest->getType() == IceType_f64);
    841   Asm->veord(Dest, getSrc(0), getSrc(1));
    842   assert(!Asm->needsTextFixup());
    843 }
    844 
    845 template <> void InstARM32Vmla::emitIAS(const Cfg *Func) const {
    846   // Note: Dest == getSrc(0) for four address FP instructions.
    847   assert(getSrcSize() == 3);
    848   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    849   const Variable *Dest = getDest();
    850   switch (Dest->getType()) {
    851   default:
    852     // TODO(kschimpf) Figure out how vector operations apply.
    853     emitUsingTextFixup(Func);
    854     return;
    855   case IceType_f32:
    856     Asm->vmlas(getDest(), getSrc(1), getSrc(2), CondARM32::AL);
    857     assert(!Asm->needsTextFixup());
    858     return;
    859   case IceType_f64:
    860     Asm->vmlad(getDest(), getSrc(1), getSrc(2), CondARM32::AL);
    861     assert(!Asm->needsTextFixup());
    862     return;
    863   }
    864 }
    865 
    866 template <> void InstARM32Vmls::emitIAS(const Cfg *Func) const {
    867   // Note: Dest == getSrc(0) for four address FP instructions.
    868   assert(getSrcSize() == 3);
    869   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    870   const Variable *Dest = getDest();
    871   switch (Dest->getType()) {
    872   default:
    873     // TODO(kschimpf) Figure out how vector operations apply.
    874     emitUsingTextFixup(Func);
    875     return;
    876   case IceType_f32:
    877     Asm->vmlss(getDest(), getSrc(1), getSrc(2), CondARM32::AL);
    878     assert(!Asm->needsTextFixup());
    879     return;
    880   case IceType_f64:
    881     Asm->vmlsd(getDest(), getSrc(1), getSrc(2), CondARM32::AL);
    882     assert(!Asm->needsTextFixup());
    883     return;
    884   }
    885 }
    886 
    887 template <> void InstARM32Vmvn::emitIAS(const Cfg *Func) const {
    888   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    889   const Variable *Dest = getDest();
    890   switch (Dest->getType()) {
    891   default:
    892     llvm::report_fatal_error("Vmvn not defined on type " +
    893                              typeStdString(Dest->getType()));
    894   case IceType_v4i1:
    895   case IceType_v8i1:
    896   case IceType_v16i1:
    897   case IceType_v16i8:
    898   case IceType_v8i16:
    899   case IceType_v4i32:
    900   case IceType_v4f32: {
    901     Asm->vmvnq(Dest, getSrc(0));
    902   } break;
    903   }
    904 }
    905 
    906 template <> void InstARM32Vneg::emitIAS(const Cfg *Func) const {
    907   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    908   const Variable *Dest = getDest();
    909   const Type DestTy = Dest->getType();
    910   switch (Dest->getType()) {
    911   default:
    912     llvm::report_fatal_error("Vneg not defined on type " +
    913                              typeStdString(Dest->getType()));
    914   case IceType_v4i1:
    915   case IceType_v8i1:
    916   case IceType_v16i1:
    917   case IceType_v16i8:
    918   case IceType_v8i16:
    919   case IceType_v4i32:
    920   case IceType_v4f32: {
    921     const Type ElmtTy = typeElementType(DestTy);
    922     Asm->vnegqs(ElmtTy, Dest, getSrc(0));
    923   } break;
    924   }
    925 }
    926 
    927 template <> void InstARM32Vorr::emitIAS(const Cfg *Func) const {
    928   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    929   const Variable *Dest = getDest();
    930   switch (Dest->getType()) {
    931   default:
    932     llvm::report_fatal_error("Vorr not defined on type " +
    933                              typeStdString(Dest->getType()));
    934   case IceType_v4i1:
    935   case IceType_v8i1:
    936   case IceType_v16i1:
    937   case IceType_v16i8:
    938   case IceType_v8i16:
    939   case IceType_v4i32:
    940     Asm->vorrq(Dest, getSrc(0), getSrc(1));
    941   }
    942   assert(!Asm->needsTextFixup());
    943 }
    944 
    945 template <> void InstARM32Vshl::emitIAS(const Cfg *Func) const {
    946   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    947   const Variable *Dest = getDest();
    948   const Type DestTy = Dest->getType();
    949   switch (DestTy) {
    950   default:
    951     llvm::report_fatal_error("Vshl not defined on type " +
    952                              typeStdString(Dest->getType()));
    953   // TODO(jpp): handle i1 vectors in terms of element count instead of element
    954   // type.
    955   case IceType_v4i1:
    956   case IceType_v8i1:
    957   case IceType_v16i1:
    958   case IceType_v16i8:
    959   case IceType_v8i16:
    960   case IceType_v4i32: {
    961     const Type ElmtTy = typeElementType(DestTy);
    962     assert(Sign != InstARM32::FS_None);
    963     switch (Sign) {
    964     case InstARM32::FS_None: // defaults to unsigned.
    965     case InstARM32::FS_Unsigned:
    966       if (const auto *Imm6 = llvm::dyn_cast<ConstantInteger32>(getSrc(1))) {
    967         Asm->vshlqc(ElmtTy, Dest, getSrc(0), Imm6);
    968       } else {
    969         Asm->vshlqu(ElmtTy, Dest, getSrc(0), getSrc(1));
    970       }
    971       break;
    972     case InstARM32::FS_Signed:
    973       if (const auto *Imm6 = llvm::dyn_cast<ConstantInteger32>(getSrc(1))) {
    974         Asm->vshlqc(ElmtTy, Dest, getSrc(0), Imm6);
    975       } else {
    976         Asm->vshlqi(ElmtTy, Dest, getSrc(0), getSrc(1));
    977       }
    978       break;
    979     }
    980   } break;
    981   }
    982 }
    983 
    984 template <> void InstARM32Vshr::emitIAS(const Cfg *Func) const {
    985   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
    986   const Variable *Dest = getDest();
    987   const Type DestTy = Dest->getType();
    988   switch (DestTy) {
    989   default:
    990     llvm::report_fatal_error("Vshr not defined on type " +
    991                              typeStdString(Dest->getType()));
    992   // TODO(jpp): handle i1 vectors in terms of element count instead of element
    993   // type.
    994   case IceType_v4i1:
    995   case IceType_v8i1:
    996   case IceType_v16i1:
    997   case IceType_v16i8:
    998   case IceType_v8i16:
    999   case IceType_v4i32: {
   1000     const Type ElmtTy = typeElementType(DestTy);
   1001     const auto *Imm6 = llvm::cast<ConstantInteger32>(getSrc(1));
   1002     assert(Sign != InstARM32::FS_None);
   1003     switch (Sign) {
   1004     case InstARM32::FS_None: // defaults to unsigned.
   1005     case InstARM32::FS_Unsigned:
   1006       Asm->vshrquc(ElmtTy, Dest, getSrc(0), Imm6);
   1007       break;
   1008     case InstARM32::FS_Signed:
   1009       Asm->vshrqic(ElmtTy, Dest, getSrc(0), Imm6);
   1010       break;
   1011     }
   1012   } break;
   1013   }
   1014 }
   1015 
   1016 template <> void InstARM32Vsub::emitIAS(const Cfg *Func) const {
   1017   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   1018   const Variable *Dest = getDest();
   1019   Type DestTy = Dest->getType();
   1020   switch (DestTy) {
   1021   default:
   1022     llvm::report_fatal_error("Vsub not defined on type " +
   1023                              typeStdString(DestTy));
   1024   case IceType_v16i8:
   1025   case IceType_v8i16:
   1026   case IceType_v4i32:
   1027     Asm->vsubqi(typeElementType(DestTy), Dest, getSrc(0), getSrc(1));
   1028     break;
   1029   case IceType_v4f32:
   1030     Asm->vsubqf(Dest, getSrc(0), getSrc(1));
   1031     break;
   1032   case IceType_f32:
   1033     Asm->vsubs(getDest(), getSrc(0), getSrc(1), CondARM32::AL);
   1034     break;
   1035   case IceType_f64:
   1036     Asm->vsubd(getDest(), getSrc(0), getSrc(1), CondARM32::AL);
   1037     break;
   1038   }
   1039   assert(!Asm->needsTextFixup());
   1040 }
   1041 
   1042 template <> void InstARM32Vmul::emitIAS(const Cfg *Func) const {
   1043   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   1044   const Variable *Dest = getDest();
   1045   const Type DestTy = Dest->getType();
   1046   switch (DestTy) {
   1047   default:
   1048     llvm::report_fatal_error("Vmul not defined on type " +
   1049                              typeStdString(DestTy));
   1050 
   1051   case IceType_v16i8:
   1052   case IceType_v8i16:
   1053   case IceType_v4i32:
   1054     Asm->vmulqi(typeElementType(DestTy), Dest, getSrc(0), getSrc(1));
   1055     break;
   1056   case IceType_v4f32:
   1057     Asm->vmulqf(Dest, getSrc(0), getSrc(1));
   1058     break;
   1059   case IceType_f32:
   1060     Asm->vmuls(Dest, getSrc(0), getSrc(1), CondARM32::AL);
   1061     break;
   1062   case IceType_f64:
   1063     Asm->vmuld(Dest, getSrc(0), getSrc(1), CondARM32::AL);
   1064     break;
   1065   }
   1066 }
   1067 
   1068 InstARM32Call::InstARM32Call(Cfg *Func, Variable *Dest, Operand *CallTarget)
   1069     : InstARM32(Func, InstARM32::Call, 1, Dest) {
   1070   HasSideEffects = true;
   1071   addSource(CallTarget);
   1072 }
   1073 
   1074 InstARM32Label::InstARM32Label(Cfg *Func, TargetARM32 *Target)
   1075     : InstARM32(Func, InstARM32::Label, 0, nullptr),
   1076       Number(Target->makeNextLabelNumber()) {
   1077   if (BuildDefs::dump()) {
   1078     Name = GlobalString::createWithString(
   1079         Func->getContext(),
   1080         ".L" + Func->getFunctionName() + "$local$__" + std::to_string(Number));
   1081   } else {
   1082     Name = GlobalString::createWithoutString(Func->getContext());
   1083   }
   1084 }
   1085 
   1086 namespace {
   1087 // Requirements for Push/Pop:
   1088 //  1) All the Variables have the same type;
   1089 //  2) All the variables have registers assigned to them.
   1090 void validatePushOrPopRegisterListOrDie(const VarList &RegList) {
   1091   Type PreviousTy = IceType_void;
   1092   for (Variable *Reg : RegList) {
   1093     if (PreviousTy != IceType_void && Reg->getType() != PreviousTy) {
   1094       llvm::report_fatal_error("Type mismatch when popping/pushing "
   1095                                "registers.");
   1096     }
   1097 
   1098     if (!Reg->hasReg()) {
   1099       llvm::report_fatal_error("Push/pop operand does not have a register "
   1100                                "assigned to it.");
   1101     }
   1102 
   1103     PreviousTy = Reg->getType();
   1104   }
   1105 }
   1106 } // end of anonymous namespace
   1107 
   1108 void InstARM32RegisterStackOp::emit(const Cfg *Func) const {
   1109   if (!BuildDefs::dump())
   1110     return;
   1111   emitUsingForm(Func, Emit_Text);
   1112 }
   1113 
   1114 void InstARM32RegisterStackOp::emitIAS(const Cfg *Func) const {
   1115   emitUsingForm(Func, Emit_Binary);
   1116   assert(!Func->getAssembler<ARM32::AssemblerARM32>()->needsTextFixup());
   1117 }
   1118 
   1119 void InstARM32RegisterStackOp::dump(const Cfg *Func) const {
   1120   if (!BuildDefs::dump())
   1121     return;
   1122   Ostream &Str = Func->getContext()->getStrDump();
   1123   Str << getDumpOpcode() << " ";
   1124   SizeT NumRegs = getNumStackRegs();
   1125   for (SizeT I = 0; I < NumRegs; ++I) {
   1126     if (I > 0)
   1127       Str << ", ";
   1128     getStackReg(I)->dump(Func);
   1129   }
   1130 }
   1131 
   1132 void InstARM32RegisterStackOp::emitGPRsAsText(const Cfg *Func) const {
   1133   if (!BuildDefs::dump())
   1134     return;
   1135   Ostream &Str = Func->getContext()->getStrEmit();
   1136   Str << "\t" << getGPROpcode() << "\t{";
   1137   getStackReg(0)->emit(Func);
   1138   const SizeT NumRegs = getNumStackRegs();
   1139   for (SizeT i = 1; i < NumRegs; ++i) {
   1140     Str << ", ";
   1141     getStackReg(i)->emit(Func);
   1142   }
   1143   Str << "}";
   1144 }
   1145 
   1146 void InstARM32RegisterStackOp::emitSRegsAsText(const Cfg *Func,
   1147                                                const Variable *BaseReg,
   1148                                                SizeT RegCount) const {
   1149   if (!BuildDefs::dump())
   1150     return;
   1151   Ostream &Str = Func->getContext()->getStrEmit();
   1152   Str << "\t" << getSRegOpcode() << "\t{";
   1153   bool IsFirst = true;
   1154   const auto Base = BaseReg->getRegNum();
   1155   for (SizeT i = 0; i < RegCount; ++i) {
   1156     if (IsFirst)
   1157       IsFirst = false;
   1158     else
   1159       Str << ", ";
   1160     Str << RegARM32::getRegName(RegNumT::fixme(Base + i));
   1161   }
   1162   Str << "}";
   1163 }
   1164 
   1165 void InstARM32RegisterStackOp::emitSRegsOp(const Cfg *Func, EmitForm Form,
   1166                                            const Variable *BaseReg,
   1167                                            SizeT RegCount,
   1168                                            SizeT InstIndex) const {
   1169   if (Form == Emit_Text && BuildDefs::dump() && InstIndex > 0) {
   1170     startNextInst(Func);
   1171     Func->getContext()->getStrEmit() << "\n";
   1172   }
   1173   emitSRegs(Func, Form, BaseReg, RegCount);
   1174 }
   1175 
   1176 namespace {
   1177 
   1178 bool isAssignedConsecutiveRegisters(const Variable *Before,
   1179                                     const Variable *After) {
   1180   assert(Before->hasReg());
   1181   assert(After->hasReg());
   1182   return RegNumT::fixme(Before->getRegNum() + 1) == After->getRegNum();
   1183 }
   1184 
   1185 } // end of anonymous namespace
   1186 
   1187 void InstARM32RegisterStackOp::emitUsingForm(const Cfg *Func,
   1188                                              const EmitForm Form) const {
   1189   SizeT NumRegs = getNumStackRegs();
   1190   assert(NumRegs);
   1191 
   1192   const auto *Reg = llvm::cast<Variable>(getStackReg(0));
   1193   if (isScalarIntegerType(Reg->getType())) {
   1194     // Push/pop GPR registers.
   1195     SizeT IntegerCount = 0;
   1196     ARM32::IValueT GPRegisters = 0;
   1197     const Variable *LastDest = nullptr;
   1198     for (SizeT i = 0; i < NumRegs; ++i) {
   1199       const Variable *Var = getStackReg(i);
   1200       assert(Var->hasReg() && "stack op only applies to registers");
   1201       const RegARM32::GPRRegister Reg =
   1202           RegARM32::getEncodedGPR(Var->getRegNum());
   1203       LastDest = Var;
   1204       GPRegisters |= (1 << Reg);
   1205       ++IntegerCount;
   1206     }
   1207     if (IntegerCount == 1) {
   1208       emitSingleGPR(Func, Form, LastDest);
   1209     } else {
   1210       emitMultipleGPRs(Func, Form, GPRegisters);
   1211     }
   1212     return;
   1213   }
   1214 
   1215   // Push/pop floating point registers. Divide into a list of instructions,
   1216   // defined on consecutive register ranges. Then generate the corresponding
   1217   // instructions.
   1218 
   1219   // Typical max number of registers ranges pushed/popd is no more than 5.
   1220   llvm::SmallVector<std::pair<const Variable *, SizeT>, 5> InstData;
   1221   const Variable *BaseReg = nullptr;
   1222   SizeT RegCount = 0;
   1223   for (SizeT i = 0; i < NumRegs; ++i) {
   1224     const Variable *NextReg = getStackReg(i);
   1225     assert(NextReg->hasReg());
   1226     if (BaseReg == nullptr) {
   1227       BaseReg = NextReg;
   1228       RegCount = 1;
   1229     } else if (RegCount < VpushVpopMaxConsecRegs &&
   1230                isAssignedConsecutiveRegisters(Reg, NextReg)) {
   1231       ++RegCount;
   1232     } else {
   1233       InstData.emplace_back(BaseReg, RegCount);
   1234       BaseReg = NextReg;
   1235       RegCount = 1;
   1236     }
   1237     Reg = NextReg;
   1238   }
   1239   if (RegCount) {
   1240     InstData.emplace_back(BaseReg, RegCount);
   1241   }
   1242   SizeT InstCount = 0;
   1243   if (llvm::isa<InstARM32Push>(*this)) {
   1244     for (const auto &Pair : InstData)
   1245       emitSRegsOp(Func, Form, Pair.first, Pair.second, InstCount++);
   1246     return;
   1247   }
   1248   assert(llvm::isa<InstARM32Pop>(*this));
   1249   for (const auto &Pair : reverse_range(InstData))
   1250     emitSRegsOp(Func, Form, Pair.first, Pair.second, InstCount++);
   1251 }
   1252 
   1253 InstARM32Pop::InstARM32Pop(Cfg *Func, const VarList &Dests)
   1254     : InstARM32RegisterStackOp(Func, InstARM32::Pop, 0, nullptr), Dests(Dests) {
   1255   // Track modifications to Dests separately via FakeDefs. Also, a pop
   1256   // instruction affects the stack pointer and so it should not be allowed to
   1257   // be automatically dead-code eliminated. This is automatic since we leave
   1258   // the Dest as nullptr.
   1259   validatePushOrPopRegisterListOrDie(Dests);
   1260 }
   1261 
   1262 InstARM32Push::InstARM32Push(Cfg *Func, const VarList &Srcs)
   1263     : InstARM32RegisterStackOp(Func, InstARM32::Push, Srcs.size(), nullptr) {
   1264   validatePushOrPopRegisterListOrDie(Srcs);
   1265   for (Variable *Source : Srcs) {
   1266     addSource(Source);
   1267   }
   1268 }
   1269 
   1270 InstARM32Ret::InstARM32Ret(Cfg *Func, Variable *LR, Variable *Source)
   1271     : InstARM32(Func, InstARM32::Ret, Source ? 2 : 1, nullptr) {
   1272   addSource(LR);
   1273   if (Source)
   1274     addSource(Source);
   1275 }
   1276 
   1277 InstARM32Str::InstARM32Str(Cfg *Func, Variable *Value, OperandARM32Mem *Mem,
   1278                            CondARM32::Cond Predicate)
   1279     : InstARM32Pred(Func, InstARM32::Str, 2, nullptr, Predicate) {
   1280   addSource(Value);
   1281   addSource(Mem);
   1282 }
   1283 
   1284 InstARM32Strex::InstARM32Strex(Cfg *Func, Variable *Dest, Variable *Value,
   1285                                OperandARM32Mem *Mem, CondARM32::Cond Predicate)
   1286     : InstARM32Pred(Func, InstARM32::Strex, 2, Dest, Predicate) {
   1287   addSource(Value);
   1288   addSource(Mem);
   1289 }
   1290 
   1291 InstARM32Trap::InstARM32Trap(Cfg *Func)
   1292     : InstARM32(Func, InstARM32::Trap, 0, nullptr) {}
   1293 
   1294 InstARM32Umull::InstARM32Umull(Cfg *Func, Variable *DestLo, Variable *DestHi,
   1295                                Variable *Src0, Variable *Src1,
   1296                                CondARM32::Cond Predicate)
   1297     : InstARM32Pred(Func, InstARM32::Umull, 2, DestLo, Predicate),
   1298       // DestHi is expected to have a FakeDef inserted by the lowering code.
   1299       DestHi(DestHi) {
   1300   addSource(Src0);
   1301   addSource(Src1);
   1302 }
   1303 
   1304 InstARM32Vcvt::InstARM32Vcvt(Cfg *Func, Variable *Dest, Variable *Src,
   1305                              VcvtVariant Variant, CondARM32::Cond Predicate)
   1306     : InstARM32Pred(Func, InstARM32::Vcvt, 1, Dest, Predicate),
   1307       Variant(Variant) {
   1308   addSource(Src);
   1309 }
   1310 
   1311 InstARM32Mov::InstARM32Mov(Cfg *Func, Variable *Dest, Operand *Src,
   1312                            CondARM32::Cond Predicate)
   1313     : InstARM32Pred(Func, InstARM32::Mov, 2, Dest, Predicate) {
   1314   auto *Dest64 = llvm::dyn_cast<Variable64On32>(Dest);
   1315   auto *Src64 = llvm::dyn_cast<Variable64On32>(Src);
   1316 
   1317   assert(Dest64 == nullptr || Src64 == nullptr);
   1318 
   1319   if (Dest64 != nullptr) {
   1320     // this-> is needed below because there is a parameter named Dest.
   1321     this->Dest = Dest64->getLo();
   1322     DestHi = Dest64->getHi();
   1323   }
   1324 
   1325   if (Src64 == nullptr) {
   1326     addSource(Src);
   1327   } else {
   1328     addSource(Src64->getLo());
   1329     addSource(Src64->getHi());
   1330   }
   1331 }
   1332 
   1333 namespace {
   1334 
   1335 // These next two functions find the D register that maps to the half of the Q
   1336 // register that this instruction is accessing.
   1337 Register getDRegister(const Variable *Src, uint32_t Index) {
   1338   assert(Src->hasReg());
   1339   const auto SrcReg = Src->getRegNum();
   1340 
   1341   const RegARM32::RegTableType &SrcEntry = RegARM32::RegTable[SrcReg];
   1342   assert(SrcEntry.IsVec128);
   1343 
   1344   const uint32_t NumElements = typeNumElements(Src->getType());
   1345 
   1346   // This code assumes the Aliases list goes Q_n, S_2n, S_2n+1. The asserts in
   1347   // the next two branches help to check that this is still true.
   1348   if (Index < NumElements / 2) {
   1349     // We have a Q register that's made up of two D registers. This assert is
   1350     // to help ensure that we picked the right D register.
   1351     //
   1352     // TODO(jpp): find a way to do this that doesn't rely on ordering of the
   1353     // alias list.
   1354     assert(RegARM32::RegTable[SrcEntry.Aliases[1]].Encoding + 1 ==
   1355            RegARM32::RegTable[SrcEntry.Aliases[2]].Encoding);
   1356     return static_cast<Register>(SrcEntry.Aliases[1]);
   1357   } else {
   1358     // We have a Q register that's made up of two D registers. This assert is
   1359     // to help ensure that we picked the right D register.
   1360     //
   1361     // TODO(jpp): find a way to do this that doesn't rely on ordering of the
   1362     // alias list.
   1363     assert(RegARM32::RegTable[SrcEntry.Aliases[2]].Encoding - 1 ==
   1364            RegARM32::RegTable[SrcEntry.Aliases[1]].Encoding);
   1365     return static_cast<Register>(SrcEntry.Aliases[2]);
   1366   }
   1367 }
   1368 
   1369 uint32_t adjustDIndex(Type Ty, uint32_t DIndex) {
   1370   // If Ty is a vector of i1, we may need to adjust DIndex. This is needed
   1371   // because, e.g., the second i1 in a v4i1 is accessed with a
   1372   //
   1373   // vmov.s8 Qd[4], Rn
   1374   switch (Ty) {
   1375   case IceType_v4i1:
   1376     return DIndex * 4;
   1377   case IceType_v8i1:
   1378     return DIndex * 2;
   1379   case IceType_v16i1:
   1380     return DIndex;
   1381   default:
   1382     return DIndex;
   1383   }
   1384 }
   1385 
   1386 uint32_t getDIndex(Type Ty, uint32_t NumElements, uint32_t Index) {
   1387   const uint32_t DIndex =
   1388       (Index < NumElements / 2) ? Index : Index - (NumElements / 2);
   1389   return adjustDIndex(Ty, DIndex);
   1390 }
   1391 
   1392 // For floating point values, we can insertelement or extractelement by moving
   1393 // directly from an S register. This function finds the right one.
   1394 Register getSRegister(const Variable *Src, uint32_t Index) {
   1395   assert(Src->hasReg());
   1396   const auto SrcReg = Src->getRegNum();
   1397 
   1398   // For floating point values, we need to be allocated to Q0 - Q7, so we can
   1399   // directly access the value we want as one of the S registers.
   1400   assert(Src->getType() == IceType_v4f32);
   1401   assert(SrcReg < RegARM32::Reg_q8);
   1402 
   1403   // This part assumes the register alias list goes q0, d0, d1, s0, s1, s2, s3.
   1404   assert(Index < 4);
   1405 
   1406   // TODO(jpp): find a way to do this that doesn't rely on ordering of the alias
   1407   // list.
   1408   return static_cast<Register>(RegARM32::RegTable[SrcReg].Aliases[Index + 3]);
   1409 }
   1410 
   1411 } // end of anonymous namespace
   1412 
   1413 void InstARM32Extract::emit(const Cfg *Func) const {
   1414   Ostream &Str = Func->getContext()->getStrEmit();
   1415   const Type DestTy = getDest()->getType();
   1416 
   1417   const auto *Src = llvm::cast<Variable>(getSrc(0));
   1418 
   1419   if (isIntegerType(DestTy)) {
   1420     Str << "\t"
   1421         << "vmov" << getPredicate();
   1422     const uint32_t BitSize = typeWidthInBytes(DestTy) * CHAR_BIT;
   1423     if (BitSize < 32) {
   1424       Str << ".s" << BitSize;
   1425     } else {
   1426       Str << "." << BitSize;
   1427     }
   1428     Str << "\t";
   1429     getDest()->emit(Func);
   1430     Str << ", ";
   1431 
   1432     const Type SrcTy = Src->getType();
   1433     const size_t VectorSize = typeNumElements(SrcTy);
   1434 
   1435     const Register SrcReg = getDRegister(Src, Index);
   1436 
   1437     Str << RegARM32::RegTable[SrcReg].Name;
   1438     Str << "[" << getDIndex(SrcTy, VectorSize, Index) << "]";
   1439   } else if (isFloatingType(DestTy)) {
   1440     const Register SrcReg = getSRegister(Src, Index);
   1441 
   1442     Str << "\t"
   1443         << "vmov" << getPredicate() << ".f32"
   1444         << "\t";
   1445     getDest()->emit(Func);
   1446     Str << ", " << RegARM32::RegTable[SrcReg].Name;
   1447   } else {
   1448     assert(false && "Invalid extract type");
   1449   }
   1450 }
   1451 
   1452 void InstARM32Extract::emitIAS(const Cfg *Func) const {
   1453   const Operand *Dest = getDest();
   1454   const Type DestTy = Dest->getType();
   1455   const Operand *Src = getSrc(0);
   1456   const Type SrcTy = Src->getType();
   1457   assert(isVectorType(Src->getType()));
   1458   assert(DestTy == typeElementType(Src->getType()));
   1459   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   1460   if (isIntegerType(DestTy)) {
   1461     Asm->vmovrqi(Dest, Src, adjustDIndex(SrcTy, Index), getPredicate());
   1462     assert(!Asm->needsTextFixup());
   1463     return;
   1464   }
   1465   assert(isFloatingType(DestTy));
   1466   Asm->vmovsqi(Dest, Src, Index, getPredicate());
   1467   assert(!Asm->needsTextFixup());
   1468 }
   1469 
   1470 namespace {
   1471 Type insertionType(Type Ty) {
   1472   assert(isVectorType(Ty));
   1473   switch (Ty) {
   1474   case IceType_v4i1:
   1475     return IceType_v4i32;
   1476   case IceType_v8i1:
   1477     return IceType_v8i16;
   1478   case IceType_v16i1:
   1479     return IceType_v16i8;
   1480   default:
   1481     return Ty;
   1482   }
   1483 }
   1484 } // end of anonymous namespace
   1485 
   1486 void InstARM32Insert::emit(const Cfg *Func) const {
   1487   Ostream &Str = Func->getContext()->getStrEmit();
   1488   const Variable *Dest = getDest();
   1489   const auto *Src = llvm::cast<Variable>(getSrc(0));
   1490   const Type DestTy = insertionType(getDest()->getType());
   1491   assert(isVectorType(DestTy));
   1492 
   1493   if (isIntegerType(DestTy)) {
   1494     Str << "\t"
   1495         << "vmov" << getPredicate();
   1496     const size_t BitSize = typeWidthInBytes(typeElementType(DestTy)) * CHAR_BIT;
   1497     Str << "." << BitSize << "\t";
   1498 
   1499     const size_t VectorSize = typeNumElements(DestTy);
   1500     const Register DestReg = getDRegister(Dest, Index);
   1501     const uint32_t Index =
   1502         getDIndex(insertionType(DestTy), VectorSize, this->Index);
   1503     Str << RegARM32::RegTable[DestReg].Name;
   1504     Str << "[" << Index << "], ";
   1505     Src->emit(Func);
   1506   } else if (isFloatingType(DestTy)) {
   1507     Str << "\t"
   1508         << "vmov" << getPredicate() << ".f32"
   1509         << "\t";
   1510     const Register DestReg = getSRegister(Dest, Index);
   1511     Str << RegARM32::RegTable[DestReg].Name << ", ";
   1512     Src->emit(Func);
   1513   } else {
   1514     assert(false && "Invalid insert type");
   1515   }
   1516 }
   1517 
   1518 void InstARM32Insert::emitIAS(const Cfg *Func) const {
   1519   const Variable *Dest = getDest();
   1520   const auto *Src = llvm::cast<Variable>(getSrc(0));
   1521   const Type DestTy = insertionType(Dest->getType());
   1522   const Type SrcTy = typeElementType(DestTy);
   1523   assert(SrcTy == Src->getType() || Src->getType() == IceType_i1);
   1524   assert(isVectorType(DestTy));
   1525   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   1526   if (isIntegerType(SrcTy)) {
   1527     Asm->vmovqir(Dest->asType(Func, DestTy, Dest->getRegNum()),
   1528                  adjustDIndex(DestTy, Index),
   1529                  Src->asType(Func, SrcTy, Src->getRegNum()), getPredicate());
   1530     assert(!Asm->needsTextFixup());
   1531     return;
   1532   }
   1533   assert(isFloatingType(SrcTy));
   1534   Asm->vmovqis(Dest, Index, Src, getPredicate());
   1535   assert(!Asm->needsTextFixup());
   1536 }
   1537 
   1538 template <InstARM32::InstKindARM32 K>
   1539 void InstARM32CmpLike<K>::emitIAS(const Cfg *Func) const {
   1540   emitUsingTextFixup(Func);
   1541 }
   1542 
   1543 template <> void InstARM32Cmn::emitIAS(const Cfg *Func) const {
   1544   assert(getSrcSize() == 2);
   1545   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   1546   Asm->cmn(getSrc(0), getSrc(1), getPredicate());
   1547   if (Asm->needsTextFixup())
   1548     emitUsingTextFixup(Func);
   1549 }
   1550 
   1551 template <> void InstARM32Cmp::emitIAS(const Cfg *Func) const {
   1552   assert(getSrcSize() == 2);
   1553   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   1554   Asm->cmp(getSrc(0), getSrc(1), getPredicate());
   1555   if (Asm->needsTextFixup())
   1556     emitUsingTextFixup(Func);
   1557 }
   1558 
   1559 template <> void InstARM32Tst::emitIAS(const Cfg *Func) const {
   1560   assert(getSrcSize() == 2);
   1561   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   1562   Asm->tst(getSrc(0), getSrc(1), getPredicate());
   1563   if (Asm->needsTextFixup())
   1564     emitUsingTextFixup(Func);
   1565 }
   1566 
   1567 InstARM32Dmb::InstARM32Dmb(Cfg *Func)
   1568     : InstARM32Pred(Func, InstARM32::Dmb, 0, nullptr, CondARM32::AL) {}
   1569 
   1570 InstARM32Nop::InstARM32Nop(Cfg *Func)
   1571     : InstARM32Pred(Func, InstARM32::Nop, 0, nullptr, CondARM32::AL) {}
   1572 
   1573 InstARM32Vcmp::InstARM32Vcmp(Cfg *Func, Variable *Src0, Operand *Src1,
   1574                              CondARM32::Cond Predicate)
   1575     : InstARM32Pred(Func, InstARM32::Vcmp, 2, nullptr, Predicate) {
   1576   HasSideEffects = true;
   1577   addSource(Src0);
   1578   addSource(Src1);
   1579 }
   1580 
   1581 InstARM32Vmrs::InstARM32Vmrs(Cfg *Func, CondARM32::Cond Predicate)
   1582     : InstARM32Pred(Func, InstARM32::Vmrs, 0, nullptr, Predicate) {
   1583   HasSideEffects = true;
   1584 }
   1585 
   1586 InstARM32Vabs::InstARM32Vabs(Cfg *Func, Variable *Dest, Variable *Src,
   1587                              CondARM32::Cond Predicate)
   1588     : InstARM32Pred(Func, InstARM32::Vabs, 1, Dest, Predicate) {
   1589   addSource(Src);
   1590 }
   1591 
   1592 // ======================== Dump routines ======================== //
   1593 
   1594 // Two-addr ops
   1595 template <> const char *InstARM32Movt::Opcode = "movt";
   1596 // Unary ops
   1597 template <> const char *InstARM32Movw::Opcode = "movw";
   1598 template <> const char *InstARM32Clz::Opcode = "clz";
   1599 template <> const char *InstARM32Mvn::Opcode = "mvn";
   1600 template <> const char *InstARM32Rbit::Opcode = "rbit";
   1601 template <> const char *InstARM32Rev::Opcode = "rev";
   1602 template <> const char *InstARM32Sxt::Opcode = "sxt"; // still requires b/h
   1603 template <> const char *InstARM32Uxt::Opcode = "uxt"; // still requires b/h
   1604 // FP
   1605 template <> const char *InstARM32Vsqrt::Opcode = "vsqrt";
   1606 // Mov-like ops
   1607 template <> const char *InstARM32Ldr::Opcode = "ldr";
   1608 template <> const char *InstARM32Ldrex::Opcode = "ldrex";
   1609 // Three-addr ops
   1610 template <> const char *InstARM32Adc::Opcode = "adc";
   1611 template <> const char *InstARM32Add::Opcode = "add";
   1612 template <> const char *InstARM32And::Opcode = "and";
   1613 template <> const char *InstARM32Asr::Opcode = "asr";
   1614 template <> const char *InstARM32Bic::Opcode = "bic";
   1615 template <> const char *InstARM32Eor::Opcode = "eor";
   1616 template <> const char *InstARM32Lsl::Opcode = "lsl";
   1617 template <> const char *InstARM32Lsr::Opcode = "lsr";
   1618 template <> const char *InstARM32Mul::Opcode = "mul";
   1619 template <> const char *InstARM32Orr::Opcode = "orr";
   1620 template <> const char *InstARM32Rsb::Opcode = "rsb";
   1621 template <> const char *InstARM32Rsc::Opcode = "rsc";
   1622 template <> const char *InstARM32Sbc::Opcode = "sbc";
   1623 template <> const char *InstARM32Sdiv::Opcode = "sdiv";
   1624 template <> const char *InstARM32Sub::Opcode = "sub";
   1625 template <> const char *InstARM32Udiv::Opcode = "udiv";
   1626 // FP
   1627 template <> const char *InstARM32Vadd::Opcode = "vadd";
   1628 template <> const char *InstARM32Vand::Opcode = "vand";
   1629 template <> const char *InstARM32Vbsl::Opcode = "vbsl";
   1630 template <> const char *InstARM32Vceq::Opcode = "vceq";
   1631 template <> const char *InstARM32ThreeAddrFP<InstARM32::Vcge>::Opcode = "vcge";
   1632 template <> const char *InstARM32ThreeAddrFP<InstARM32::Vcgt>::Opcode = "vcgt";
   1633 template <> const char *InstARM32Vdiv::Opcode = "vdiv";
   1634 template <> const char *InstARM32Veor::Opcode = "veor";
   1635 template <> const char *InstARM32Vmla::Opcode = "vmla";
   1636 template <> const char *InstARM32Vmls::Opcode = "vmls";
   1637 template <> const char *InstARM32Vmul::Opcode = "vmul";
   1638 template <> const char *InstARM32Vmvn::Opcode = "vmvn";
   1639 template <> const char *InstARM32Vorr::Opcode = "vorr";
   1640 template <> const char *InstARM32UnaryopFP<InstARM32::Vneg>::Opcode = "vneg";
   1641 template <> const char *InstARM32ThreeAddrFP<InstARM32::Vshl>::Opcode = "vshl";
   1642 template <> const char *InstARM32ThreeAddrFP<InstARM32::Vshr>::Opcode = "vshr";
   1643 template <> const char *InstARM32Vsub::Opcode = "vsub";
   1644 // Four-addr ops
   1645 template <> const char *InstARM32Mla::Opcode = "mla";
   1646 template <> const char *InstARM32Mls::Opcode = "mls";
   1647 // Cmp-like ops
   1648 template <> const char *InstARM32Cmn::Opcode = "cmn";
   1649 template <> const char *InstARM32Cmp::Opcode = "cmp";
   1650 template <> const char *InstARM32Tst::Opcode = "tst";
   1651 
   1652 void InstARM32::dump(const Cfg *Func) const {
   1653   if (!BuildDefs::dump())
   1654     return;
   1655   Ostream &Str = Func->getContext()->getStrDump();
   1656   Str << "[ARM32] ";
   1657   Inst::dump(Func);
   1658 }
   1659 
   1660 void InstARM32Mov::emitMultiDestSingleSource(const Cfg *Func) const {
   1661   if (!BuildDefs::dump())
   1662     return;
   1663   Ostream &Str = Func->getContext()->getStrEmit();
   1664   Variable *DestLo = getDest();
   1665   Variable *DestHi = getDestHi();
   1666   auto *Src = llvm::cast<Variable>(getSrc(0));
   1667 
   1668   assert(DestHi->hasReg());
   1669   assert(DestLo->hasReg());
   1670   assert(Src->hasReg());
   1671 
   1672   Str << "\t"
   1673          "vmov" << getPredicate() << "\t";
   1674   DestLo->emit(Func);
   1675   Str << ", ";
   1676   DestHi->emit(Func);
   1677   Str << ", ";
   1678   Src->emit(Func);
   1679 }
   1680 
   1681 void InstARM32Mov::emitSingleDestMultiSource(const Cfg *Func) const {
   1682   if (!BuildDefs::dump())
   1683     return;
   1684   Ostream &Str = Func->getContext()->getStrEmit();
   1685   Variable *Dest = getDest();
   1686   auto *SrcLo = llvm::cast<Variable>(getSrc(0));
   1687   auto *SrcHi = llvm::cast<Variable>(getSrc(1));
   1688 
   1689   assert(SrcHi->hasReg());
   1690   assert(SrcLo->hasReg());
   1691   assert(Dest->hasReg());
   1692   assert(getSrcSize() == 2);
   1693 
   1694   Str << "\t"
   1695          "vmov" << getPredicate() << "\t";
   1696   Dest->emit(Func);
   1697   Str << ", ";
   1698   SrcLo->emit(Func);
   1699   Str << ", ";
   1700   SrcHi->emit(Func);
   1701 }
   1702 
   1703 namespace {
   1704 
   1705 bool isVariableWithoutRegister(const Operand *Op) {
   1706   if (const auto *OpV = llvm::dyn_cast<Variable>(Op)) {
   1707     return !OpV->hasReg();
   1708   }
   1709   return false;
   1710 }
   1711 bool isMemoryAccess(Operand *Op) {
   1712   return isVariableWithoutRegister(Op) || llvm::isa<OperandARM32Mem>(Op);
   1713 }
   1714 
   1715 bool isMoveBetweenCoreAndVFPRegisters(Variable *Dest, Operand *Src) {
   1716   const Type DestTy = Dest->getType();
   1717   const Type SrcTy = Src->getType();
   1718   return !isVectorType(DestTy) && !isVectorType(SrcTy) &&
   1719          (isScalarIntegerType(DestTy) == isScalarFloatingType(SrcTy));
   1720 }
   1721 
   1722 } // end of anonymous namespace
   1723 
   1724 void InstARM32Mov::emitSingleDestSingleSource(const Cfg *Func) const {
   1725   if (!BuildDefs::dump())
   1726     return;
   1727   Ostream &Str = Func->getContext()->getStrEmit();
   1728   Variable *Dest = getDest();
   1729 
   1730   if (!Dest->hasReg()) {
   1731     llvm::report_fatal_error("mov can't store.");
   1732   }
   1733 
   1734   Operand *Src0 = getSrc(0);
   1735   if (isMemoryAccess(Src0)) {
   1736     llvm::report_fatal_error("mov can't load.");
   1737   }
   1738 
   1739   Type Ty = Dest->getType();
   1740   const bool IsVector = isVectorType(Ty);
   1741   const bool IsScalarFP = isScalarFloatingType(Ty);
   1742   const bool CoreVFPMove = isMoveBetweenCoreAndVFPRegisters(Dest, Src0);
   1743   const bool IsVMove = (IsVector || IsScalarFP || CoreVFPMove);
   1744   const char *Opcode = IsVMove ? "vmov" : "mov";
   1745   // when vmov{c}'ing, we need to emit a width string. Otherwise, the
   1746   // assembler might be tempted to assume we want a vector vmov{c}, and that
   1747   // is disallowed because ARM.
   1748   const char *WidthString = !CoreVFPMove ? getFpWidthString(Ty) : "";
   1749   CondARM32::Cond Cond = getPredicate();
   1750   if (IsVector)
   1751     assert(CondARM32::isUnconditional(Cond) &&
   1752            "Moves on vectors must be unconditional!");
   1753   Str << "\t" << Opcode;
   1754   if (IsVMove) {
   1755     Str << Cond << WidthString;
   1756   } else {
   1757     Str << WidthString << Cond;
   1758   }
   1759   Str << "\t";
   1760   Dest->emit(Func);
   1761   Str << ", ";
   1762   Src0->emit(Func);
   1763 }
   1764 
   1765 void InstARM32Mov::emit(const Cfg *Func) const {
   1766   if (!BuildDefs::dump())
   1767     return;
   1768   assert(!(isMultiDest() && isMultiSource()) && "Invalid vmov type.");
   1769   if (isMultiDest()) {
   1770     emitMultiDestSingleSource(Func);
   1771     return;
   1772   }
   1773 
   1774   if (isMultiSource()) {
   1775     emitSingleDestMultiSource(Func);
   1776     return;
   1777   }
   1778 
   1779   emitSingleDestSingleSource(Func);
   1780 }
   1781 
   1782 void InstARM32Mov::emitIAS(const Cfg *Func) const {
   1783   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   1784   const Variable *Dest = getDest();
   1785   Operand *Src0 = getSrc(0);
   1786   const CondARM32::Cond Cond = getPredicate();
   1787   if (!Dest->hasReg()) {
   1788     llvm::report_fatal_error("mov can't store.");
   1789   }
   1790   if (isMemoryAccess(Src0)) {
   1791     llvm::report_fatal_error("mov can't load.");
   1792   }
   1793 
   1794   assert(!(isMultiDest() && isMultiSource()) && "Invalid vmov type.");
   1795   if (isMultiDest()) {
   1796     Asm->vmovrrd(Dest, getDestHi(), Src0, Cond);
   1797     return;
   1798   }
   1799   if (isMultiSource()) {
   1800     Asm->vmovdrr(Dest, Src0, getSrc(1), Cond);
   1801     return;
   1802   }
   1803 
   1804   const Type DestTy = Dest->getType();
   1805   const Type SrcTy = Src0->getType();
   1806   switch (DestTy) {
   1807   default:
   1808     break; // Error
   1809   case IceType_i1:
   1810   case IceType_i8:
   1811   case IceType_i16:
   1812   case IceType_i32:
   1813     switch (SrcTy) {
   1814     default:
   1815       break; // Error
   1816     case IceType_i1:
   1817     case IceType_i8:
   1818     case IceType_i16:
   1819     case IceType_i32:
   1820     case IceType_i64:
   1821       Asm->mov(Dest, Src0, Cond);
   1822       return;
   1823     case IceType_f32:
   1824       Asm->vmovrs(Dest, Src0, Cond);
   1825       return;
   1826     }
   1827     break; // Error
   1828   case IceType_i64:
   1829     if (isScalarIntegerType(SrcTy)) {
   1830       Asm->mov(Dest, Src0, Cond);
   1831       return;
   1832     }
   1833     if (SrcTy == IceType_f64) {
   1834       if (const auto *Var = llvm::dyn_cast<Variable>(Src0)) {
   1835         Asm->vmovdd(Dest, Var, Cond);
   1836         return;
   1837       }
   1838       if (const auto *FpImm = llvm::dyn_cast<OperandARM32FlexFpImm>(Src0)) {
   1839         Asm->vmovd(Dest, FpImm, Cond);
   1840         return;
   1841       }
   1842     }
   1843     break; // Error
   1844   case IceType_f32:
   1845     switch (SrcTy) {
   1846     default:
   1847       break; // Error
   1848     case IceType_i1:
   1849     case IceType_i8:
   1850     case IceType_i16:
   1851     case IceType_i32:
   1852       return Asm->vmovsr(Dest, Src0, Cond);
   1853     case IceType_f32:
   1854       if (const auto *Var = llvm::dyn_cast<Variable>(Src0)) {
   1855         Asm->vmovss(Dest, Var, Cond);
   1856         return;
   1857       }
   1858       if (const auto *FpImm = llvm::dyn_cast<OperandARM32FlexFpImm>(Src0)) {
   1859         Asm->vmovs(Dest, FpImm, Cond);
   1860         return;
   1861       }
   1862       break; // Error
   1863     }
   1864     break; // Error
   1865   case IceType_f64:
   1866     if (SrcTy == IceType_f64) {
   1867       if (const auto *Var = llvm::dyn_cast<Variable>(Src0)) {
   1868         Asm->vmovdd(Dest, Var, Cond);
   1869         return;
   1870       }
   1871       if (const auto *FpImm = llvm::dyn_cast<OperandARM32FlexFpImm>(Src0)) {
   1872         Asm->vmovd(Dest, FpImm, Cond);
   1873         return;
   1874       }
   1875     }
   1876     break; // Error
   1877   // TODO(jpp): Remove vectors of i1.
   1878   case IceType_v4i1:
   1879   case IceType_v8i1:
   1880   case IceType_v16i1:
   1881   case IceType_v16i8:
   1882   case IceType_v8i16:
   1883   case IceType_v4i32:
   1884   case IceType_v4f32:
   1885     assert(CondARM32::isUnconditional(Cond) &&
   1886            "Moves on vector must be unconditional!");
   1887     if (isVectorType(SrcTy)) {
   1888       // Mov between different Src and Dest types is used for bitcasting
   1889       // vectors.  We still want to make sure SrcTy is a vector type.
   1890       Asm->vorrq(Dest, Src0, Src0);
   1891       return;
   1892     } else if (const auto *C = llvm::dyn_cast<ConstantInteger32>(Src0)) {
   1893       // Mov with constant argument, allowing the initializing all elements of
   1894       // the vector.
   1895       if (Asm->vmovqc(Dest, C))
   1896         return;
   1897     }
   1898   }
   1899   llvm::report_fatal_error("Mov: don't know how to move " +
   1900                            typeStdString(SrcTy) + " to " +
   1901                            typeStdString(DestTy));
   1902 }
   1903 
   1904 void InstARM32Mov::dump(const Cfg *Func) const {
   1905   if (!BuildDefs::dump())
   1906     return;
   1907   assert(getSrcSize() == 1 || getSrcSize() == 2);
   1908   Ostream &Str = Func->getContext()->getStrDump();
   1909   Variable *Dest = getDest();
   1910   Variable *DestHi = getDestHi();
   1911   Dest->dump(Func);
   1912   if (DestHi) {
   1913     Str << ", ";
   1914     DestHi->dump(Func);
   1915   }
   1916 
   1917   dumpOpcodePred(Str, " = mov", getDest()->getType());
   1918   Str << " ";
   1919 
   1920   dumpSources(Func);
   1921 }
   1922 
   1923 void InstARM32Br::emit(const Cfg *Func) const {
   1924   if (!BuildDefs::dump())
   1925     return;
   1926   Ostream &Str = Func->getContext()->getStrEmit();
   1927   Str << "\t"
   1928          "b" << getPredicate() << "\t";
   1929   if (Label) {
   1930     Str << Label->getLabelName();
   1931   } else {
   1932     if (isUnconditionalBranch()) {
   1933       Str << getTargetFalse()->getAsmName();
   1934     } else {
   1935       Str << getTargetTrue()->getAsmName();
   1936       if (getTargetFalse()) {
   1937         startNextInst(Func);
   1938         Str << "\n\t"
   1939             << "b"
   1940             << "\t" << getTargetFalse()->getAsmName();
   1941       }
   1942     }
   1943   }
   1944 }
   1945 
   1946 void InstARM32Br::emitIAS(const Cfg *Func) const {
   1947   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   1948   if (Label) {
   1949     Asm->b(Asm->getOrCreateLocalLabel(Label->getNumber()), getPredicate());
   1950   } else if (isUnconditionalBranch()) {
   1951     Asm->b(Asm->getOrCreateCfgNodeLabel(getTargetFalse()->getIndex()),
   1952            getPredicate());
   1953   } else {
   1954     Asm->b(Asm->getOrCreateCfgNodeLabel(getTargetTrue()->getIndex()),
   1955            getPredicate());
   1956     if (const CfgNode *False = getTargetFalse())
   1957       Asm->b(Asm->getOrCreateCfgNodeLabel(False->getIndex()), CondARM32::AL);
   1958   }
   1959   if (Asm->needsTextFixup())
   1960     emitUsingTextFixup(Func);
   1961 }
   1962 
   1963 void InstARM32Br::dump(const Cfg *Func) const {
   1964   if (!BuildDefs::dump())
   1965     return;
   1966   Ostream &Str = Func->getContext()->getStrDump();
   1967   Str << "br ";
   1968 
   1969   if (getPredicate() == CondARM32::AL) {
   1970     if (Label) {
   1971       Str << "label %" << Label->getLabelName();
   1972     } else {
   1973       Str << "label %" << getTargetFalse()->getName();
   1974     }
   1975     return;
   1976   }
   1977 
   1978   if (Label) {
   1979     Str << getPredicate() << ", label %" << Label->getLabelName();
   1980   } else {
   1981     Str << getPredicate() << ", label %" << getTargetTrue()->getName();
   1982     if (getTargetFalse()) {
   1983       Str << ", label %" << getTargetFalse()->getName();
   1984     }
   1985   }
   1986 }
   1987 
   1988 void InstARM32Call::emit(const Cfg *Func) const {
   1989   if (!BuildDefs::dump())
   1990     return;
   1991   Ostream &Str = Func->getContext()->getStrEmit();
   1992   assert(getSrcSize() == 1);
   1993   if (llvm::isa<ConstantInteger32>(getCallTarget())) {
   1994     // This shouldn't happen (typically have to copy the full 32-bits to a
   1995     // register and do an indirect jump).
   1996     llvm::report_fatal_error("ARM32Call to ConstantInteger32");
   1997   } else if (const auto *CallTarget =
   1998                  llvm::dyn_cast<ConstantRelocatable>(getCallTarget())) {
   1999     // Calls only have 24-bits, but the linker should insert veneers to extend
   2000     // the range if needed.
   2001     Str << "\t"
   2002            "bl"
   2003            "\t";
   2004     CallTarget->emitWithoutPrefix(Func->getTarget());
   2005   } else {
   2006     Str << "\t"
   2007            "blx"
   2008            "\t";
   2009     getCallTarget()->emit(Func);
   2010   }
   2011 }
   2012 
   2013 void InstARM32Call::emitIAS(const Cfg *Func) const {
   2014   assert(getSrcSize() == 1);
   2015   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2016   if (llvm::isa<ConstantInteger32>(getCallTarget())) {
   2017     // This shouldn't happen (typically have to copy the full 32-bits to a
   2018     // register and do an indirect jump).
   2019     llvm::report_fatal_error("ARM32Call to ConstantInteger32");
   2020   } else if (const auto *CallTarget =
   2021                  llvm::dyn_cast<ConstantRelocatable>(getCallTarget())) {
   2022     // Calls only have 24-bits, but the linker should insert veneers to extend
   2023     // the range if needed.
   2024     Asm->bl(CallTarget);
   2025   } else {
   2026     Asm->blx(getCallTarget());
   2027   }
   2028   if (Asm->needsTextFixup())
   2029     return emitUsingTextFixup(Func);
   2030 }
   2031 
   2032 void InstARM32Call::dump(const Cfg *Func) const {
   2033   if (!BuildDefs::dump())
   2034     return;
   2035   Ostream &Str = Func->getContext()->getStrDump();
   2036   if (getDest()) {
   2037     dumpDest(Func);
   2038     Str << " = ";
   2039   }
   2040   Str << "call ";
   2041   getCallTarget()->dump(Func);
   2042 }
   2043 
   2044 void InstARM32Label::emit(const Cfg *Func) const {
   2045   if (!BuildDefs::dump())
   2046     return;
   2047   // A label is not really an instruction. Hence, we need to fix the
   2048   // emitted text size.
   2049   if (auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>())
   2050     Asm->decEmitTextSize(InstSize);
   2051   Ostream &Str = Func->getContext()->getStrEmit();
   2052   Str << getLabelName() << ":";
   2053 }
   2054 
   2055 void InstARM32Label::emitIAS(const Cfg *Func) const {
   2056   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2057   Asm->bindLocalLabel(this, Number);
   2058   if (OffsetReloc != nullptr) {
   2059     Asm->bindRelocOffset(OffsetReloc);
   2060   }
   2061   if (Asm->needsTextFixup())
   2062     emitUsingTextFixup(Func);
   2063 }
   2064 
   2065 void InstARM32Label::dump(const Cfg *Func) const {
   2066   if (!BuildDefs::dump())
   2067     return;
   2068   Ostream &Str = Func->getContext()->getStrDump();
   2069   Str << getLabelName() << ":";
   2070 }
   2071 
   2072 template <InstARM32::InstKindARM32 K>
   2073 void InstARM32LoadBase<K>::emitIAS(const Cfg *Func) const {
   2074   emitUsingTextFixup(Func);
   2075 }
   2076 
   2077 template <> void InstARM32Ldr::emit(const Cfg *Func) const {
   2078   if (!BuildDefs::dump())
   2079     return;
   2080   Ostream &Str = Func->getContext()->getStrEmit();
   2081   assert(getSrcSize() == 1);
   2082   assert(getDest()->hasReg());
   2083   Variable *Dest = getDest();
   2084   Type Ty = Dest->getType();
   2085   const bool IsVector = isVectorType(Ty);
   2086   const bool IsScalarFloat = isScalarFloatingType(Ty);
   2087   const char *ActualOpcode =
   2088       IsVector ? "vld1" : (IsScalarFloat ? "vldr" : "ldr");
   2089   const char *WidthString = IsVector ? "" : getWidthString(Ty);
   2090   Str << "\t" << ActualOpcode;
   2091   const bool IsVInst = IsVector || IsScalarFloat;
   2092   if (IsVInst) {
   2093     Str << getPredicate() << WidthString;
   2094   } else {
   2095     Str << WidthString << getPredicate();
   2096   }
   2097   if (IsVector)
   2098     Str << "." << getVecElmtBitsize(Ty);
   2099   Str << "\t";
   2100   getDest()->emit(Func);
   2101   Str << ", ";
   2102   getSrc(0)->emit(Func);
   2103 }
   2104 
   2105 template <> void InstARM32Ldr::emitIAS(const Cfg *Func) const {
   2106   assert(getSrcSize() == 1);
   2107   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2108   Variable *Dest = getDest();
   2109   const Type DestTy = Dest->getType();
   2110   switch (DestTy) {
   2111   default:
   2112     llvm::report_fatal_error("Ldr on unknown type: " + typeStdString(DestTy));
   2113   case IceType_i1:
   2114   case IceType_i8:
   2115   case IceType_i16:
   2116   case IceType_i32:
   2117   case IceType_i64:
   2118     Asm->ldr(Dest, getSrc(0), getPredicate(), Func->getTarget());
   2119     break;
   2120   case IceType_f32:
   2121     Asm->vldrs(Dest, getSrc(0), getPredicate(), Func->getTarget());
   2122     break;
   2123   case IceType_f64:
   2124     Asm->vldrd(Dest, getSrc(0), getPredicate(), Func->getTarget());
   2125     break;
   2126   case IceType_v16i8:
   2127   case IceType_v8i16:
   2128   case IceType_v4i32:
   2129   case IceType_v4f32:
   2130   case IceType_v16i1:
   2131   case IceType_v8i1:
   2132   case IceType_v4i1:
   2133     Asm->vld1qr(getVecElmtBitsize(DestTy), Dest, getSrc(0), Func->getTarget());
   2134     break;
   2135   }
   2136 }
   2137 
   2138 template <> void InstARM32Ldrex::emit(const Cfg *Func) const {
   2139   if (!BuildDefs::dump())
   2140     return;
   2141   Ostream &Str = Func->getContext()->getStrEmit();
   2142   assert(getSrcSize() == 1);
   2143   assert(getDest()->hasReg());
   2144   Variable *Dest = getDest();
   2145   Type DestTy = Dest->getType();
   2146   assert(isScalarIntegerType(DestTy));
   2147   const char *WidthString = getWidthString(DestTy);
   2148   Str << "\t" << Opcode << WidthString << getPredicate() << "\t";
   2149   getDest()->emit(Func);
   2150   Str << ", ";
   2151   getSrc(0)->emit(Func);
   2152 }
   2153 
   2154 template <> void InstARM32Ldrex::emitIAS(const Cfg *Func) const {
   2155   assert(getSrcSize() == 1);
   2156   assert(getDest()->hasReg());
   2157   Variable *Dest = getDest();
   2158   assert(isScalarIntegerType(Dest->getType()));
   2159   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2160   Asm->ldrex(Dest, getSrc(0), getPredicate(), Func->getTarget());
   2161   if (Asm->needsTextFixup())
   2162     emitUsingTextFixup(Func);
   2163 }
   2164 
   2165 template <InstARM32::InstKindARM32 K>
   2166 void InstARM32TwoAddrGPR<K>::emitIAS(const Cfg *Func) const {
   2167   emitUsingTextFixup(Func);
   2168 }
   2169 
   2170 template <InstARM32::InstKindARM32 K, bool Nws>
   2171 void InstARM32UnaryopGPR<K, Nws>::emitIAS(const Cfg *Func) const {
   2172   emitUsingTextFixup(Func);
   2173 }
   2174 
   2175 template <> void InstARM32Rbit::emitIAS(const Cfg *Func) const {
   2176   assert(getSrcSize() == 1);
   2177   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2178   Asm->rbit(getDest(), getSrc(0), getPredicate());
   2179   if (Asm->needsTextFixup())
   2180     emitUsingTextFixup(Func);
   2181 }
   2182 
   2183 template <> void InstARM32Rev::emitIAS(const Cfg *Func) const {
   2184   assert(getSrcSize() == 1);
   2185   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2186   Asm->rev(getDest(), getSrc(0), getPredicate());
   2187   if (Asm->needsTextFixup())
   2188     emitUsingTextFixup(Func);
   2189 }
   2190 
   2191 template <> void InstARM32Movw::emit(const Cfg *Func) const {
   2192   if (!BuildDefs::dump())
   2193     return;
   2194   Ostream &Str = Func->getContext()->getStrEmit();
   2195   assert(getSrcSize() == 1);
   2196   Str << "\t" << Opcode << getPredicate() << "\t";
   2197   getDest()->emit(Func);
   2198   Str << ", ";
   2199   auto *Src0 = llvm::cast<Constant>(getSrc(0));
   2200   if (auto *CR = llvm::dyn_cast<ConstantRelocatable>(Src0)) {
   2201     Str << "#:lower16:";
   2202     CR->emitWithoutPrefix(Func->getTarget());
   2203     if (getFlags().getUseNonsfi()) {
   2204       Str << " - .";
   2205     }
   2206   } else {
   2207     Src0->emit(Func);
   2208   }
   2209 }
   2210 
   2211 template <> void InstARM32Movw::emitIAS(const Cfg *Func) const {
   2212   assert(getSrcSize() == 1);
   2213   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2214   Asm->movw(getDest(), getSrc(0), getPredicate());
   2215   if (Asm->needsTextFixup())
   2216     emitUsingTextFixup(Func);
   2217 }
   2218 
   2219 template <> void InstARM32Movt::emit(const Cfg *Func) const {
   2220   if (!BuildDefs::dump())
   2221     return;
   2222   Ostream &Str = Func->getContext()->getStrEmit();
   2223   assert(getSrcSize() == 2);
   2224   Variable *Dest = getDest();
   2225   auto *Src1 = llvm::cast<Constant>(getSrc(1));
   2226   Str << "\t" << Opcode << getPredicate() << "\t";
   2227   Dest->emit(Func);
   2228   Str << ", ";
   2229   if (auto *CR = llvm::dyn_cast<ConstantRelocatable>(Src1)) {
   2230     Str << "#:upper16:";
   2231     CR->emitWithoutPrefix(Func->getTarget());
   2232     if (getFlags().getUseNonsfi()) {
   2233       Str << " - .";
   2234     }
   2235   } else {
   2236     Src1->emit(Func);
   2237   }
   2238 }
   2239 
   2240 template <> void InstARM32Movt::emitIAS(const Cfg *Func) const {
   2241   assert(getSrcSize() == 2);
   2242   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2243   Asm->movt(getDest(), getSrc(1), getPredicate());
   2244   if (Asm->needsTextFixup())
   2245     emitUsingTextFixup(Func);
   2246 }
   2247 
   2248 template <> void InstARM32Clz::emitIAS(const Cfg *Func) const {
   2249   assert(getSrcSize() == 1);
   2250   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2251   Asm->clz(getDest(), getSrc(0), getPredicate());
   2252   if (Asm->needsTextFixup())
   2253     emitUsingTextFixup(Func);
   2254 }
   2255 
   2256 template <> void InstARM32Mvn::emitIAS(const Cfg *Func) const {
   2257   assert(getSrcSize() == 1);
   2258   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2259   Asm->mvn(getDest(), getSrc(0), getPredicate());
   2260   if (Asm->needsTextFixup())
   2261     emitUsingTextFixup(Func);
   2262 }
   2263 
   2264 template <> void InstARM32Sxt::emitIAS(const Cfg *Func) const {
   2265   assert(getSrcSize() == 1);
   2266   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2267   Asm->sxt(getDest(), getSrc(0), getPredicate());
   2268   if (Asm->needsTextFixup())
   2269     emitUsingTextFixup(Func);
   2270 }
   2271 
   2272 template <> void InstARM32Uxt::emitIAS(const Cfg *Func) const {
   2273   assert(getSrcSize() == 1);
   2274   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2275   Asm->uxt(getDest(), getSrc(0), getPredicate());
   2276   if (Asm->needsTextFixup())
   2277     emitUsingTextFixup(Func);
   2278 }
   2279 
   2280 template <InstARM32::InstKindARM32 K>
   2281 void InstARM32UnaryopFP<K>::emitIAS(const Cfg *Func) const {
   2282   emitUsingTextFixup(Func);
   2283 }
   2284 
   2285 template <InstARM32::InstKindARM32 K>
   2286 void InstARM32UnaryopSignAwareFP<K>::emitIAS(const Cfg *Func) const {
   2287   InstARM32::emitUsingTextFixup(Func);
   2288 }
   2289 
   2290 template <> void InstARM32Vsqrt::emitIAS(const Cfg *Func) const {
   2291   assert(getSrcSize() == 1);
   2292   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2293   const Operand *Dest = getDest();
   2294   switch (Dest->getType()) {
   2295   case IceType_f32:
   2296     Asm->vsqrts(Dest, getSrc(0), getPredicate());
   2297     break;
   2298   case IceType_f64:
   2299     Asm->vsqrtd(Dest, getSrc(0), getPredicate());
   2300     break;
   2301   default:
   2302     llvm::report_fatal_error("Vsqrt of non-floating type");
   2303   }
   2304   if (Asm->needsTextFixup())
   2305     emitUsingTextFixup(Func);
   2306 }
   2307 
   2308 const char *InstARM32Pop::getGPROpcode() const { return "pop"; }
   2309 
   2310 const char *InstARM32Pop::getSRegOpcode() const { return "vpop"; }
   2311 
   2312 Variable *InstARM32Pop::getStackReg(SizeT Index) const { return Dests[Index]; }
   2313 
   2314 SizeT InstARM32Pop::getNumStackRegs() const { return Dests.size(); }
   2315 
   2316 void InstARM32Pop::emitSingleGPR(const Cfg *Func, const EmitForm Form,
   2317                                  const Variable *Reg) const {
   2318   switch (Form) {
   2319   case Emit_Text:
   2320     emitGPRsAsText(Func);
   2321     return;
   2322   case Emit_Binary:
   2323     Func->getAssembler<ARM32::AssemblerARM32>()->pop(Reg, CondARM32::AL);
   2324     return;
   2325   }
   2326 }
   2327 
   2328 void InstARM32Pop::emitMultipleGPRs(const Cfg *Func, const EmitForm Form,
   2329                                     IValueT Registers) const {
   2330   switch (Form) {
   2331   case Emit_Text:
   2332     emitGPRsAsText(Func);
   2333     return;
   2334   case Emit_Binary:
   2335     Func->getAssembler<ARM32::AssemblerARM32>()->popList(Registers,
   2336                                                          CondARM32::AL);
   2337     return;
   2338   }
   2339 }
   2340 
   2341 void InstARM32Pop::emitSRegs(const Cfg *Func, const EmitForm Form,
   2342                              const Variable *BaseReg, SizeT RegCount) const {
   2343   switch (Form) {
   2344   case Emit_Text:
   2345     emitSRegsAsText(Func, BaseReg, RegCount);
   2346     return;
   2347   case Emit_Binary:
   2348     Func->getAssembler<ARM32::AssemblerARM32>()->vpop(BaseReg, RegCount,
   2349                                                       CondARM32::AL);
   2350     return;
   2351   }
   2352 }
   2353 
   2354 const char *InstARM32Push::getGPROpcode() const { return "push"; }
   2355 
   2356 const char *InstARM32Push::getSRegOpcode() const { return "vpush"; }
   2357 
   2358 Variable *InstARM32Push::getStackReg(SizeT Index) const {
   2359   return llvm::cast<Variable>(getSrc(Index));
   2360 }
   2361 
   2362 SizeT InstARM32Push::getNumStackRegs() const { return getSrcSize(); }
   2363 
   2364 void InstARM32Push::emitSingleGPR(const Cfg *Func, const EmitForm Form,
   2365                                   const Variable *Reg) const {
   2366   switch (Form) {
   2367   case Emit_Text:
   2368     emitGPRsAsText(Func);
   2369     return;
   2370   case Emit_Binary:
   2371     Func->getAssembler<ARM32::AssemblerARM32>()->push(Reg, CondARM32::AL);
   2372     return;
   2373   }
   2374 }
   2375 
   2376 void InstARM32Push::emitMultipleGPRs(const Cfg *Func, const EmitForm Form,
   2377                                      IValueT Registers) const {
   2378   switch (Form) {
   2379   case Emit_Text:
   2380     emitGPRsAsText(Func);
   2381     return;
   2382   case Emit_Binary:
   2383     Func->getAssembler<ARM32::AssemblerARM32>()->pushList(Registers,
   2384                                                           CondARM32::AL);
   2385     return;
   2386   }
   2387 }
   2388 
   2389 void InstARM32Push::emitSRegs(const Cfg *Func, const EmitForm Form,
   2390                               const Variable *BaseReg, SizeT RegCount) const {
   2391   switch (Form) {
   2392   case Emit_Text:
   2393     emitSRegsAsText(Func, BaseReg, RegCount);
   2394     return;
   2395   case Emit_Binary:
   2396     Func->getAssembler<ARM32::AssemblerARM32>()->vpush(BaseReg, RegCount,
   2397                                                        CondARM32::AL);
   2398     return;
   2399   }
   2400 }
   2401 
   2402 void InstARM32Ret::emit(const Cfg *Func) const {
   2403   if (!BuildDefs::dump())
   2404     return;
   2405   assert(getSrcSize() > 0);
   2406   auto *LR = llvm::cast<Variable>(getSrc(0));
   2407   assert(LR->hasReg());
   2408   assert(LR->getRegNum() == RegARM32::Reg_lr);
   2409   Ostream &Str = Func->getContext()->getStrEmit();
   2410   Str << "\t"
   2411          "bx"
   2412          "\t";
   2413   LR->emit(Func);
   2414 }
   2415 
   2416 void InstARM32Ret::emitIAS(const Cfg *Func) const {
   2417   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2418   Asm->bx(RegARM32::Encoded_Reg_lr);
   2419   if (Asm->needsTextFixup())
   2420     emitUsingTextFixup(Func);
   2421 }
   2422 
   2423 void InstARM32Ret::dump(const Cfg *Func) const {
   2424   if (!BuildDefs::dump())
   2425     return;
   2426   Ostream &Str = Func->getContext()->getStrDump();
   2427   Type Ty = (getSrcSize() == 1 ? IceType_void : getSrc(0)->getType());
   2428   Str << "ret." << Ty << " ";
   2429   dumpSources(Func);
   2430 }
   2431 
   2432 void InstARM32Str::emit(const Cfg *Func) const {
   2433   if (!BuildDefs::dump())
   2434     return;
   2435   Ostream &Str = Func->getContext()->getStrEmit();
   2436   assert(getSrcSize() == 2);
   2437   Type Ty = getSrc(0)->getType();
   2438   const bool IsVectorStore = isVectorType(Ty);
   2439   const bool IsScalarFloat = isScalarFloatingType(Ty);
   2440   const char *Opcode =
   2441       IsVectorStore ? "vst1" : (IsScalarFloat ? "vstr" : "str");
   2442   Str << "\t" << Opcode;
   2443   const bool IsVInst = IsVectorStore || IsScalarFloat;
   2444   if (IsVInst) {
   2445     Str << getPredicate() << getWidthString(Ty);
   2446   } else {
   2447     Str << getWidthString(Ty) << getPredicate();
   2448   }
   2449   if (IsVectorStore)
   2450     Str << "." << getVecElmtBitsize(Ty);
   2451   Str << "\t";
   2452   getSrc(0)->emit(Func);
   2453   Str << ", ";
   2454   getSrc(1)->emit(Func);
   2455 }
   2456 
   2457 void InstARM32Str::emitIAS(const Cfg *Func) const {
   2458   assert(getSrcSize() == 2);
   2459   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2460   const Operand *Src0 = getSrc(0);
   2461   const Operand *Src1 = getSrc(1);
   2462   Type Ty = Src0->getType();
   2463   switch (Ty) {
   2464   default:
   2465     llvm::report_fatal_error("Str on unknown type: " + typeStdString(Ty));
   2466   case IceType_i1:
   2467   case IceType_i8:
   2468   case IceType_i16:
   2469   case IceType_i32:
   2470   case IceType_i64:
   2471     Asm->str(Src0, Src1, getPredicate(), Func->getTarget());
   2472     break;
   2473   case IceType_f32:
   2474     Asm->vstrs(Src0, Src1, getPredicate(), Func->getTarget());
   2475     break;
   2476   case IceType_f64:
   2477     Asm->vstrd(Src0, Src1, getPredicate(), Func->getTarget());
   2478     break;
   2479   case IceType_v16i8:
   2480   case IceType_v8i16:
   2481   case IceType_v4i32:
   2482   case IceType_v4f32:
   2483   case IceType_v16i1:
   2484   case IceType_v8i1:
   2485   case IceType_v4i1:
   2486     Asm->vst1qr(getVecElmtBitsize(Ty), Src0, Src1, Func->getTarget());
   2487     break;
   2488   }
   2489 }
   2490 
   2491 void InstARM32Str::dump(const Cfg *Func) const {
   2492   if (!BuildDefs::dump())
   2493     return;
   2494   Ostream &Str = Func->getContext()->getStrDump();
   2495   Type Ty = getSrc(0)->getType();
   2496   dumpOpcodePred(Str, "str", Ty);
   2497   Str << " ";
   2498   getSrc(1)->dump(Func);
   2499   Str << ", ";
   2500   getSrc(0)->dump(Func);
   2501 }
   2502 
   2503 void InstARM32Strex::emit(const Cfg *Func) const {
   2504   if (!BuildDefs::dump())
   2505     return;
   2506   assert(getSrcSize() == 2);
   2507   Type Ty = getSrc(0)->getType();
   2508   assert(isScalarIntegerType(Ty));
   2509   Variable *Dest = getDest();
   2510   Ostream &Str = Func->getContext()->getStrEmit();
   2511   static constexpr char Opcode[] = "strex";
   2512   const char *WidthString = getWidthString(Ty);
   2513   Str << "\t" << Opcode << WidthString << getPredicate() << "\t";
   2514   Dest->emit(Func);
   2515   Str << ", ";
   2516   emitSources(Func);
   2517 }
   2518 
   2519 void InstARM32Strex::emitIAS(const Cfg *Func) const {
   2520   assert(getSrcSize() == 2);
   2521   const Operand *Src0 = getSrc(0);
   2522   assert(isScalarIntegerType(Src0->getType()));
   2523   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2524   Asm->strex(Dest, Src0, getSrc(1), getPredicate(), Func->getTarget());
   2525   if (Asm->needsTextFixup())
   2526     emitUsingTextFixup(Func);
   2527 }
   2528 
   2529 void InstARM32Strex::dump(const Cfg *Func) const {
   2530   if (!BuildDefs::dump())
   2531     return;
   2532   Ostream &Str = Func->getContext()->getStrDump();
   2533   Variable *Dest = getDest();
   2534   Dest->dump(Func);
   2535   Str << " = ";
   2536   Type Ty = getSrc(0)->getType();
   2537   dumpOpcodePred(Str, "strex", Ty);
   2538   Str << " ";
   2539   getSrc(1)->dump(Func);
   2540   Str << ", ";
   2541   getSrc(0)->dump(Func);
   2542 }
   2543 
   2544 void InstARM32Trap::emit(const Cfg *Func) const {
   2545   if (!BuildDefs::dump())
   2546     return;
   2547   Ostream &Str = Func->getContext()->getStrEmit();
   2548   assert(getSrcSize() == 0);
   2549   // There isn't a mnemonic for the special NaCl Trap encoding, so dump
   2550   // the raw bytes.
   2551   Str << "\t.long 0x";
   2552   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2553   for (uint8_t I : Asm->getNonExecBundlePadding()) {
   2554     Str.write_hex(I);
   2555   }
   2556 }
   2557 
   2558 void InstARM32Trap::emitIAS(const Cfg *Func) const {
   2559   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2560   Asm->trap();
   2561   assert(!Asm->needsTextFixup());
   2562 }
   2563 
   2564 void InstARM32Trap::dump(const Cfg *Func) const {
   2565   if (!BuildDefs::dump())
   2566     return;
   2567   Ostream &Str = Func->getContext()->getStrDump();
   2568   Str << "trap";
   2569 }
   2570 
   2571 void InstARM32Umull::emit(const Cfg *Func) const {
   2572   if (!BuildDefs::dump())
   2573     return;
   2574   Ostream &Str = Func->getContext()->getStrEmit();
   2575   assert(getSrcSize() == 2);
   2576   assert(getDest()->hasReg());
   2577   Str << "\t"
   2578          "umull" << getPredicate() << "\t";
   2579   getDest()->emit(Func);
   2580   Str << ", ";
   2581   DestHi->emit(Func);
   2582   Str << ", ";
   2583   getSrc(0)->emit(Func);
   2584   Str << ", ";
   2585   getSrc(1)->emit(Func);
   2586 }
   2587 
   2588 void InstARM32Umull::emitIAS(const Cfg *Func) const {
   2589   assert(getSrcSize() == 2);
   2590   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2591   Asm->umull(getDest(), DestHi, getSrc(0), getSrc(1), getPredicate());
   2592   if (Asm->needsTextFixup())
   2593     emitUsingTextFixup(Func);
   2594 }
   2595 
   2596 void InstARM32Umull::dump(const Cfg *Func) const {
   2597   if (!BuildDefs::dump())
   2598     return;
   2599   Ostream &Str = Func->getContext()->getStrDump();
   2600   dumpDest(Func);
   2601   Str << " = ";
   2602   dumpOpcodePred(Str, "umull", getDest()->getType());
   2603   Str << " ";
   2604   dumpSources(Func);
   2605 }
   2606 
   2607 namespace {
   2608 const char *vcvtVariantSuffix(const InstARM32Vcvt::VcvtVariant Variant) {
   2609   switch (Variant) {
   2610   case InstARM32Vcvt::S2si:
   2611     return ".s32.f32";
   2612   case InstARM32Vcvt::S2ui:
   2613     return ".u32.f32";
   2614   case InstARM32Vcvt::Si2s:
   2615     return ".f32.s32";
   2616   case InstARM32Vcvt::Ui2s:
   2617     return ".f32.u32";
   2618   case InstARM32Vcvt::D2si:
   2619     return ".s32.f64";
   2620   case InstARM32Vcvt::D2ui:
   2621     return ".u32.f64";
   2622   case InstARM32Vcvt::Si2d:
   2623     return ".f64.s32";
   2624   case InstARM32Vcvt::Ui2d:
   2625     return ".f64.u32";
   2626   case InstARM32Vcvt::S2d:
   2627     return ".f64.f32";
   2628   case InstARM32Vcvt::D2s:
   2629     return ".f32.f64";
   2630   case InstARM32Vcvt::Vs2si:
   2631     return ".s32.f32";
   2632   case InstARM32Vcvt::Vs2ui:
   2633     return ".u32.f32";
   2634   case InstARM32Vcvt::Vsi2s:
   2635     return ".f32.s32";
   2636   case InstARM32Vcvt::Vui2s:
   2637     return ".f32.u32";
   2638   }
   2639   llvm::report_fatal_error("Invalid VcvtVariant enum.");
   2640 }
   2641 } // end of anonymous namespace
   2642 
   2643 void InstARM32Vcvt::emit(const Cfg *Func) const {
   2644   if (!BuildDefs::dump())
   2645     return;
   2646   Ostream &Str = Func->getContext()->getStrEmit();
   2647   assert(getSrcSize() == 1);
   2648   assert(getDest()->hasReg());
   2649   Str << "\t"
   2650          "vcvt" << getPredicate() << vcvtVariantSuffix(Variant) << "\t";
   2651   getDest()->emit(Func);
   2652   Str << ", ";
   2653   getSrc(0)->emit(Func);
   2654 }
   2655 
   2656 void InstARM32Vcvt::emitIAS(const Cfg *Func) const {
   2657   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2658   switch (Variant) {
   2659   case S2si:
   2660     Asm->vcvtis(getDest(), getSrc(0), getPredicate());
   2661     break;
   2662   case S2ui:
   2663     Asm->vcvtus(getDest(), getSrc(0), getPredicate());
   2664     break;
   2665   case Si2s:
   2666     Asm->vcvtsi(getDest(), getSrc(0), getPredicate());
   2667     break;
   2668   case Ui2s:
   2669     Asm->vcvtsu(getDest(), getSrc(0), getPredicate());
   2670     break;
   2671   case D2si:
   2672     Asm->vcvtid(getDest(), getSrc(0), getPredicate());
   2673     break;
   2674   case D2ui:
   2675     Asm->vcvtud(getDest(), getSrc(0), getPredicate());
   2676     break;
   2677   case Si2d:
   2678     Asm->vcvtdi(getDest(), getSrc(0), getPredicate());
   2679     break;
   2680   case Ui2d:
   2681     Asm->vcvtdu(getDest(), getSrc(0), getPredicate());
   2682     break;
   2683   case S2d:
   2684     Asm->vcvtds(getDest(), getSrc(0), getPredicate());
   2685     break;
   2686   case D2s:
   2687     Asm->vcvtsd(getDest(), getSrc(0), getPredicate());
   2688     break;
   2689   case Vs2si:
   2690     Asm->vcvtqsi(getDest(), getSrc(0));
   2691     break;
   2692   case Vs2ui:
   2693     Asm->vcvtqsu(getDest(), getSrc(0));
   2694     break;
   2695   case Vsi2s:
   2696     Asm->vcvtqis(getDest(), getSrc(0));
   2697     break;
   2698   case Vui2s:
   2699     Asm->vcvtqus(getDest(), getSrc(0));
   2700     break;
   2701   }
   2702   assert(!Asm->needsTextFixup());
   2703 }
   2704 
   2705 void InstARM32Vcvt::dump(const Cfg *Func) const {
   2706   if (!BuildDefs::dump())
   2707     return;
   2708   Ostream &Str = Func->getContext()->getStrDump();
   2709   dumpDest(Func);
   2710   Str << " = "
   2711       << "vcvt" << getPredicate() << vcvtVariantSuffix(Variant) << " ";
   2712   dumpSources(Func);
   2713 }
   2714 
   2715 void InstARM32Vcmp::emit(const Cfg *Func) const {
   2716   if (!BuildDefs::dump())
   2717     return;
   2718   Ostream &Str = Func->getContext()->getStrEmit();
   2719   assert(getSrcSize() == 2);
   2720   Str << "\t"
   2721          "vcmp" << getPredicate() << getFpWidthString(getSrc(0)->getType())
   2722       << "\t";
   2723   getSrc(0)->emit(Func);
   2724   Str << ", ";
   2725   getSrc(1)->emit(Func);
   2726 }
   2727 
   2728 void InstARM32Vcmp::emitIAS(const Cfg *Func) const {
   2729   assert(getSrcSize() == 2);
   2730   const Operand *Src0 = getSrc(0);
   2731   const Type Ty = Src0->getType();
   2732   const Operand *Src1 = getSrc(1);
   2733   const CondARM32::Cond Cond = getPredicate();
   2734   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2735   if (llvm::isa<OperandARM32FlexFpZero>(Src1)) {
   2736     switch (Ty) {
   2737     case IceType_f32:
   2738       Asm->vcmpsz(Src0, Cond);
   2739       break;
   2740     case IceType_f64:
   2741       Asm->vcmpdz(Src0, Cond);
   2742       break;
   2743     default:
   2744       llvm::report_fatal_error("Vcvt on non floating value");
   2745     }
   2746   } else {
   2747     switch (Ty) {
   2748     case IceType_f32:
   2749       Asm->vcmps(Src0, Src1, Cond);
   2750       break;
   2751     case IceType_f64:
   2752       Asm->vcmpd(Src0, Src1, Cond);
   2753       break;
   2754     default:
   2755       llvm::report_fatal_error("Vcvt on non floating value");
   2756     }
   2757   }
   2758   assert(!Asm->needsTextFixup());
   2759 }
   2760 
   2761 void InstARM32Vcmp::dump(const Cfg *Func) const {
   2762   if (!BuildDefs::dump())
   2763     return;
   2764   Ostream &Str = Func->getContext()->getStrDump();
   2765   Str << "vcmp" << getPredicate() << getFpWidthString(getSrc(0)->getType());
   2766   dumpSources(Func);
   2767 }
   2768 
   2769 void InstARM32Vmrs::emit(const Cfg *Func) const {
   2770   if (!BuildDefs::dump())
   2771     return;
   2772   Ostream &Str = Func->getContext()->getStrEmit();
   2773   assert(getSrcSize() == 0);
   2774   Str << "\t"
   2775          "vmrs" << getPredicate() << "\t"
   2776                                      "APSR_nzcv"
   2777                                      ", "
   2778                                      "FPSCR";
   2779 }
   2780 
   2781 void InstARM32Vmrs::emitIAS(const Cfg *Func) const {
   2782   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2783   Asm->vmrsAPSR_nzcv(getPredicate());
   2784   assert(!Asm->needsTextFixup());
   2785 }
   2786 
   2787 void InstARM32Vmrs::dump(const Cfg *Func) const {
   2788   if (!BuildDefs::dump())
   2789     return;
   2790   Ostream &Str = Func->getContext()->getStrDump();
   2791   Str << "APSR{n,z,v,c} = vmrs" << getPredicate() << "\t"
   2792                                                      "FPSCR{n,z,c,v}";
   2793 }
   2794 
   2795 void InstARM32Vabs::emit(const Cfg *Func) const {
   2796   if (!BuildDefs::dump())
   2797     return;
   2798   Ostream &Str = Func->getContext()->getStrEmit();
   2799   assert(getSrcSize() == 1);
   2800   Str << "\t"
   2801          "vabs" << getPredicate() << getFpWidthString(getSrc(0)->getType())
   2802       << "\t";
   2803   getDest()->emit(Func);
   2804   Str << ", ";
   2805   getSrc(0)->emit(Func);
   2806 }
   2807 
   2808 void InstARM32Vabs::emitIAS(const Cfg *Func) const {
   2809   assert(getSrcSize() == 1);
   2810   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2811   const Variable *Dest = getDest();
   2812   switch (Dest->getType()) {
   2813   default:
   2814     llvm::report_fatal_error("fabs not defined on type " +
   2815                              typeStdString(Dest->getType()));
   2816   case IceType_f32:
   2817     Asm->vabss(Dest, getSrc(0), getPredicate());
   2818     break;
   2819   case IceType_f64:
   2820     Asm->vabsd(Dest, getSrc(0), getPredicate());
   2821     break;
   2822   case IceType_v4f32:
   2823     assert(CondARM32::isUnconditional(getPredicate()) &&
   2824            "fabs must be unconditional");
   2825     Asm->vabsq(Dest, getSrc(0));
   2826   }
   2827   assert(!Asm->needsTextFixup());
   2828 }
   2829 
   2830 void InstARM32Vabs::dump(const Cfg *Func) const {
   2831   if (!BuildDefs::dump())
   2832     return;
   2833   Ostream &Str = Func->getContext()->getStrDump();
   2834   dumpDest(Func);
   2835   Str << " = vabs" << getPredicate() << getFpWidthString(getSrc(0)->getType());
   2836 }
   2837 
   2838 void InstARM32Dmb::emit(const Cfg *Func) const {
   2839   if (!BuildDefs::dump())
   2840     return;
   2841   Ostream &Str = Func->getContext()->getStrEmit();
   2842   assert(getSrcSize() == 0);
   2843   Str << "\t"
   2844          "dmb"
   2845          "\t"
   2846          "sy";
   2847 }
   2848 
   2849 void InstARM32Dmb::emitIAS(const Cfg *Func) const {
   2850   assert(getSrcSize() == 0);
   2851   auto *Asm = Func->getAssembler<ARM32::AssemblerARM32>();
   2852   constexpr ARM32::IValueT SyOption = 0xF; // i.e. 1111
   2853   Asm->dmb(SyOption);
   2854   if (Asm->needsTextFixup())
   2855     emitUsingTextFixup(Func);
   2856 }
   2857 
   2858 void InstARM32Dmb::dump(const Cfg *Func) const {
   2859   if (!BuildDefs::dump())
   2860     return;
   2861   Func->getContext()->getStrDump() << "dmb\t"
   2862                                       "sy";
   2863 }
   2864 
   2865 void InstARM32Nop::emit(const Cfg *Func) const {
   2866   if (!BuildDefs::dump())
   2867     return;
   2868   assert(getSrcSize() == 0);
   2869   Func->getContext()->getStrEmit() << "\t"
   2870                                    << "nop";
   2871 }
   2872 
   2873 void InstARM32Nop::emitIAS(const Cfg *Func) const {
   2874   assert(getSrcSize() == 0);
   2875   Func->getAssembler<ARM32::AssemblerARM32>()->nop();
   2876 }
   2877 
   2878 void InstARM32Nop::dump(const Cfg *Func) const {
   2879   if (!BuildDefs::dump())
   2880     return;
   2881   assert(getSrcSize() == 0);
   2882   Func->getContext()->getStrDump() << "nop";
   2883 }
   2884 
   2885 void OperandARM32Mem::emit(const Cfg *Func) const {
   2886   if (!BuildDefs::dump())
   2887     return;
   2888   Ostream &Str = Func->getContext()->getStrEmit();
   2889   Str << "[";
   2890   getBase()->emit(Func);
   2891   switch (getAddrMode()) {
   2892   case PostIndex:
   2893   case NegPostIndex:
   2894     Str << "]";
   2895     break;
   2896   default:
   2897     break;
   2898   }
   2899   if (isRegReg()) {
   2900     Str << ", ";
   2901     if (isNegAddrMode()) {
   2902       Str << "-";
   2903     }
   2904     getIndex()->emit(Func);
   2905     if (getShiftOp() != kNoShift) {
   2906       Str << ", " << InstARM32ShiftAttributes[getShiftOp()].EmitString << " #"
   2907           << getShiftAmt();
   2908     }
   2909   } else {
   2910     ConstantInteger32 *Offset = getOffset();
   2911     if (Offset && Offset->getValue() != 0) {
   2912       Str << ", ";
   2913       Offset->emit(Func);
   2914     }
   2915   }
   2916   switch (getAddrMode()) {
   2917   case Offset:
   2918   case NegOffset:
   2919     Str << "]";
   2920     break;
   2921   case PreIndex:
   2922   case NegPreIndex:
   2923     Str << "]!";
   2924     break;
   2925   case PostIndex:
   2926   case NegPostIndex:
   2927     // Brace is already closed off.
   2928     break;
   2929   }
   2930 }
   2931 
   2932 void OperandARM32Mem::dump(const Cfg *Func, Ostream &Str) const {
   2933   if (!BuildDefs::dump())
   2934     return;
   2935   Str << "[";
   2936   if (Func)
   2937     getBase()->dump(Func);
   2938   else
   2939     getBase()->dump(Str);
   2940   Str << ", ";
   2941   if (isRegReg()) {
   2942     if (isNegAddrMode()) {
   2943       Str << "-";
   2944     }
   2945     if (Func)
   2946       getIndex()->dump(Func);
   2947     else
   2948       getIndex()->dump(Str);
   2949     if (getShiftOp() != kNoShift) {
   2950       Str << ", " << InstARM32ShiftAttributes[getShiftOp()].EmitString << " #"
   2951           << getShiftAmt();
   2952     }
   2953   } else {
   2954     getOffset()->dump(Func, Str);
   2955   }
   2956   Str << "] AddrMode==" << getAddrMode();
   2957 }
   2958 
   2959 void OperandARM32ShAmtImm::emit(const Cfg *Func) const { ShAmt->emit(Func); }
   2960 
   2961 void OperandARM32ShAmtImm::dump(const Cfg *, Ostream &Str) const {
   2962   ShAmt->dump(Str);
   2963 }
   2964 
   2965 OperandARM32FlexImm *OperandARM32FlexImm::create(Cfg *Func, Type Ty,
   2966                                                  uint32_t Imm,
   2967                                                  uint32_t RotateAmt) {
   2968   // The assembler wants the smallest rotation. Rotate if needed. Note: Imm is
   2969   // an 8-bit value.
   2970   assert(Utils::IsUint(8, Imm) &&
   2971          "Flex immediates can only be defined on 8-bit immediates");
   2972   while ((Imm & 0x03) == 0 && RotateAmt > 0) {
   2973     --RotateAmt;
   2974     Imm = Imm >> 2;
   2975   }
   2976   return new (Func->allocate<OperandARM32FlexImm>())
   2977       OperandARM32FlexImm(Func, Ty, Imm, RotateAmt);
   2978 }
   2979 
   2980 void OperandARM32FlexImm::emit(const Cfg *Func) const {
   2981   if (!BuildDefs::dump())
   2982     return;
   2983   Ostream &Str = Func->getContext()->getStrEmit();
   2984   uint32_t Imm = getImm();
   2985   uint32_t RotateAmt = getRotateAmt();
   2986   Str << "#" << Utils::rotateRight32(Imm, 2 * RotateAmt);
   2987 }
   2988 
   2989 void OperandARM32FlexImm::dump(const Cfg * /* Func */, Ostream &Str) const {
   2990   if (!BuildDefs::dump())
   2991     return;
   2992   uint32_t Imm = getImm();
   2993   uint32_t RotateAmt = getRotateAmt();
   2994   Str << "#(" << Imm << " ror 2*" << RotateAmt << ")";
   2995 }
   2996 
   2997 namespace {
   2998 static constexpr uint32_t a = 0x80;
   2999 static constexpr uint32_t b = 0x40;
   3000 static constexpr uint32_t cdefgh = 0x3F;
   3001 static constexpr uint32_t AllowedBits = a | b | cdefgh;
   3002 static_assert(AllowedBits == 0xFF,
   3003               "Invalid mask for f32/f64 constant rematerialization.");
   3004 
   3005 // There's no loss in always returning the modified immediate as float.
   3006 // TODO(jpp): returning a double causes problems when outputting the constants
   3007 // for filetype=asm. Why?
   3008 float materializeFloatImmediate(uint32_t ModifiedImm) {
   3009   const uint32_t Ret = ((ModifiedImm & a) ? 0x80000000 : 0) |
   3010                        ((ModifiedImm & b) ? 0x3E000000 : 0x40000000) |
   3011                        ((ModifiedImm & cdefgh) << 19);
   3012   return Utils::bitCopy<float>(Ret);
   3013 }
   3014 
   3015 } // end of anonymous namespace
   3016 
   3017 void OperandARM32FlexFpImm::emit(const Cfg *Func) const {
   3018   if (!BuildDefs::dump())
   3019     return;
   3020   Ostream &Str = Func->getContext()->getStrEmit();
   3021   switch (Ty) {
   3022   default:
   3023     llvm::report_fatal_error("Invalid flex fp imm type.");
   3024   case IceType_f64:
   3025   case IceType_f32:
   3026     Str << "#" << materializeFloatImmediate(ModifiedImm)
   3027         << " @ Modified: " << ModifiedImm;
   3028     break;
   3029   }
   3030 }
   3031 
   3032 void OperandARM32FlexFpImm::dump(const Cfg * /*Func*/, Ostream &Str) const {
   3033   if (!BuildDefs::dump())
   3034     return;
   3035   Str << "#" << materializeFloatImmediate(ModifiedImm) << getFpWidthString(Ty);
   3036 }
   3037 
   3038 void OperandARM32FlexFpZero::emit(const Cfg *Func) const {
   3039   if (!BuildDefs::dump())
   3040     return;
   3041   Ostream &Str = Func->getContext()->getStrEmit();
   3042   switch (Ty) {
   3043   default:
   3044     llvm::report_fatal_error("Invalid flex fp imm type.");
   3045   case IceType_f64:
   3046   case IceType_f32:
   3047     Str << "#0.0";
   3048   }
   3049 }
   3050 
   3051 void OperandARM32FlexFpZero::dump(const Cfg * /*Func*/, Ostream &Str) const {
   3052   if (!BuildDefs::dump())
   3053     return;
   3054   Str << "#0.0" << getFpWidthString(Ty);
   3055 }
   3056 
   3057 void OperandARM32FlexReg::emit(const Cfg *Func) const {
   3058   if (!BuildDefs::dump())
   3059     return;
   3060   Ostream &Str = Func->getContext()->getStrEmit();
   3061   getReg()->emit(Func);
   3062   if (getShiftOp() != kNoShift) {
   3063     Str << ", " << InstARM32ShiftAttributes[getShiftOp()].EmitString << " ";
   3064     getShiftAmt()->emit(Func);
   3065   }
   3066 }
   3067 
   3068 void OperandARM32FlexReg::dump(const Cfg *Func, Ostream &Str) const {
   3069   if (!BuildDefs::dump())
   3070     return;
   3071   Variable *Reg = getReg();
   3072   if (Func)
   3073     Reg->dump(Func);
   3074   else
   3075     Reg->dump(Str);
   3076   if (getShiftOp() != kNoShift) {
   3077     Str << ", " << InstARM32ShiftAttributes[getShiftOp()].EmitString << " ";
   3078     if (Func)
   3079       getShiftAmt()->dump(Func);
   3080     else
   3081       getShiftAmt()->dump(Str);
   3082   }
   3083 }
   3084 
   3085 // Force instantition of template classes
   3086 template class InstARM32ThreeAddrGPR<InstARM32::Adc>;
   3087 template class InstARM32ThreeAddrGPR<InstARM32::Add>;
   3088 template class InstARM32ThreeAddrGPR<InstARM32::And>;
   3089 template class InstARM32ThreeAddrGPR<InstARM32::Asr>;
   3090 template class InstARM32ThreeAddrGPR<InstARM32::Bic>;
   3091 template class InstARM32ThreeAddrGPR<InstARM32::Eor>;
   3092 template class InstARM32ThreeAddrGPR<InstARM32::Lsl>;
   3093 template class InstARM32ThreeAddrGPR<InstARM32::Lsr>;
   3094 template class InstARM32ThreeAddrGPR<InstARM32::Mul>;
   3095 template class InstARM32ThreeAddrGPR<InstARM32::Orr>;
   3096 template class InstARM32ThreeAddrGPR<InstARM32::Rsb>;
   3097 template class InstARM32ThreeAddrGPR<InstARM32::Rsc>;
   3098 template class InstARM32ThreeAddrGPR<InstARM32::Sbc>;
   3099 template class InstARM32ThreeAddrGPR<InstARM32::Sdiv>;
   3100 template class InstARM32ThreeAddrGPR<InstARM32::Sub>;
   3101 template class InstARM32ThreeAddrGPR<InstARM32::Udiv>;
   3102 
   3103 template class InstARM32ThreeAddrFP<InstARM32::Vadd>;
   3104 template class InstARM32ThreeAddrSignAwareFP<InstARM32::Vcge>;
   3105 template class InstARM32ThreeAddrSignAwareFP<InstARM32::Vcgt>;
   3106 template class InstARM32ThreeAddrFP<InstARM32::Vdiv>;
   3107 template class InstARM32ThreeAddrFP<InstARM32::Veor>;
   3108 template class InstARM32FourAddrFP<InstARM32::Vmla>;
   3109 template class InstARM32FourAddrFP<InstARM32::Vmls>;
   3110 template class InstARM32ThreeAddrFP<InstARM32::Vmul>;
   3111 template class InstARM32UnaryopSignAwareFP<InstARM32::Vneg>;
   3112 template class InstARM32ThreeAddrSignAwareFP<InstARM32::Vshl>;
   3113 template class InstARM32ThreeAddrSignAwareFP<InstARM32::Vshr>;
   3114 template class InstARM32ThreeAddrFP<InstARM32::Vsub>;
   3115 
   3116 template class InstARM32LoadBase<InstARM32::Ldr>;
   3117 template class InstARM32LoadBase<InstARM32::Ldrex>;
   3118 
   3119 template class InstARM32TwoAddrGPR<InstARM32::Movt>;
   3120 
   3121 template class InstARM32UnaryopGPR<InstARM32::Movw, false>;
   3122 template class InstARM32UnaryopGPR<InstARM32::Clz, false>;
   3123 template class InstARM32UnaryopGPR<InstARM32::Mvn, false>;
   3124 template class InstARM32UnaryopGPR<InstARM32::Rbit, false>;
   3125 template class InstARM32UnaryopGPR<InstARM32::Rev, false>;
   3126 template class InstARM32UnaryopGPR<InstARM32::Sxt, true>;
   3127 template class InstARM32UnaryopGPR<InstARM32::Uxt, true>;
   3128 template class InstARM32UnaryopFP<InstARM32::Vsqrt>;
   3129 
   3130 template class InstARM32FourAddrGPR<InstARM32::Mla>;
   3131 template class InstARM32FourAddrGPR<InstARM32::Mls>;
   3132 
   3133 template class InstARM32CmpLike<InstARM32::Cmn>;
   3134 template class InstARM32CmpLike<InstARM32::Cmp>;
   3135 template class InstARM32CmpLike<InstARM32::Tst>;
   3136 
   3137 } // end of namespace ARM32
   3138 } // end of namespace Ice
   3139