Home | History | Annotate | Download | only in IR
      1 //===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 // This file defines all of the AARCH64-specific intrinsics.
     11 //
     12 //===----------------------------------------------------------------------===//
     13 
     14 let TargetPrefix = "aarch64" in {
     15 
     16 def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
     17 def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
     18 def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>;
     19 def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>;
     20 
     21 def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>;
     22 def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>;
     23 def int_aarch64_stxp : Intrinsic<[llvm_i32_ty],
     24                                [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>;
     25 def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty],
     26                                 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>;
     27 
     28 def int_aarch64_clrex : Intrinsic<[]>;
     29 
     30 def int_aarch64_sdiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
     31                                 LLVMMatchType<0>], [IntrNoMem]>;
     32 def int_aarch64_udiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
     33                                 LLVMMatchType<0>], [IntrNoMem]>;
     34 
     35 //===----------------------------------------------------------------------===//
     36 // HINT
     37 
     38 def int_aarch64_hint : Intrinsic<[], [llvm_i32_ty]>;
     39 
     40 //===----------------------------------------------------------------------===//
     41 // Data Barrier Instructions
     42 
     43 def int_aarch64_dmb : GCCBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, Intrinsic<[], [llvm_i32_ty]>;
     44 def int_aarch64_dsb : GCCBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, Intrinsic<[], [llvm_i32_ty]>;
     45 def int_aarch64_isb : GCCBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, Intrinsic<[], [llvm_i32_ty]>;
     46 
     47 }
     48 
     49 //===----------------------------------------------------------------------===//
     50 // Advanced SIMD (NEON)
     51 
     52 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
     53   class AdvSIMD_2Scalar_Float_Intrinsic
     54     : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
     55                 [IntrNoMem]>;
     56 
     57   class AdvSIMD_FPToIntRounding_Intrinsic
     58     : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>;
     59 
     60   class AdvSIMD_1IntArg_Intrinsic
     61     : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>;
     62   class AdvSIMD_1FloatArg_Intrinsic
     63     : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
     64   class AdvSIMD_1VectorArg_Intrinsic
     65     : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>;
     66   class AdvSIMD_1VectorArg_Expand_Intrinsic
     67     : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>;
     68   class AdvSIMD_1VectorArg_Long_Intrinsic
     69     : Intrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>;
     70   class AdvSIMD_1IntArg_Narrow_Intrinsic
     71     : Intrinsic<[llvm_anyint_ty], [llvm_anyint_ty], [IntrNoMem]>;
     72   class AdvSIMD_1VectorArg_Narrow_Intrinsic
     73     : Intrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>;
     74   class AdvSIMD_1VectorArg_Int_Across_Intrinsic
     75     : Intrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>;
     76   class AdvSIMD_1VectorArg_Float_Across_Intrinsic
     77     : Intrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>;
     78 
     79   class AdvSIMD_2IntArg_Intrinsic
     80     : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
     81                 [IntrNoMem]>;
     82   class AdvSIMD_2FloatArg_Intrinsic
     83     : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
     84                 [IntrNoMem]>;
     85   class AdvSIMD_2VectorArg_Intrinsic
     86     : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
     87                 [IntrNoMem]>;
     88   class AdvSIMD_2VectorArg_Compare_Intrinsic
     89     : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>],
     90                 [IntrNoMem]>;
     91   class AdvSIMD_2Arg_FloatCompare_Intrinsic
     92     : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>],
     93                 [IntrNoMem]>;
     94   class AdvSIMD_2VectorArg_Long_Intrinsic
     95     : Intrinsic<[llvm_anyvector_ty],
     96                 [LLVMTruncatedType<0>, LLVMTruncatedType<0>],
     97                 [IntrNoMem]>;
     98   class AdvSIMD_2VectorArg_Wide_Intrinsic
     99     : Intrinsic<[llvm_anyvector_ty],
    100                 [LLVMMatchType<0>, LLVMTruncatedType<0>],
    101                 [IntrNoMem]>;
    102   class AdvSIMD_2VectorArg_Narrow_Intrinsic
    103     : Intrinsic<[llvm_anyvector_ty],
    104                 [LLVMExtendedType<0>, LLVMExtendedType<0>],
    105                 [IntrNoMem]>;
    106   class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic
    107     : Intrinsic<[llvm_anyint_ty],
    108                 [LLVMExtendedType<0>, llvm_i32_ty],
    109                 [IntrNoMem]>;
    110   class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic
    111     : Intrinsic<[llvm_anyvector_ty],
    112                 [llvm_anyvector_ty],
    113                 [IntrNoMem]>;
    114   class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic
    115     : Intrinsic<[llvm_anyvector_ty],
    116                 [LLVMTruncatedType<0>],
    117                 [IntrNoMem]>;
    118   class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic
    119     : Intrinsic<[llvm_anyvector_ty],
    120                 [LLVMTruncatedType<0>, llvm_i32_ty],
    121                 [IntrNoMem]>;
    122   class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic
    123     : Intrinsic<[llvm_anyvector_ty],
    124                 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty],
    125                 [IntrNoMem]>;
    126 
    127   class AdvSIMD_3VectorArg_Intrinsic
    128       : Intrinsic<[llvm_anyvector_ty],
    129                [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
    130                [IntrNoMem]>;
    131   class AdvSIMD_3VectorArg_Scalar_Intrinsic
    132       : Intrinsic<[llvm_anyvector_ty],
    133                [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
    134                [IntrNoMem]>;
    135   class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic
    136       : Intrinsic<[llvm_anyvector_ty],
    137                [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty,
    138                 LLVMMatchType<1>], [IntrNoMem]>;
    139   class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic
    140     : Intrinsic<[llvm_anyvector_ty],
    141                 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty],
    142                 [IntrNoMem]>;
    143   class AdvSIMD_CvtFxToFP_Intrinsic
    144     : Intrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
    145                 [IntrNoMem]>;
    146   class AdvSIMD_CvtFPToFx_Intrinsic
    147     : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty],
    148                 [IntrNoMem]>;
    149 }
    150 
    151 // Arithmetic ops
    152 
    153 let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {
    154   // Vector Add Across Lanes
    155   def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
    156   def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
    157   def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
    158 
    159   // Vector Long Add Across Lanes
    160   def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
    161   def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
    162 
    163   // Vector Halving Add
    164   def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic;
    165   def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic;
    166 
    167   // Vector Rounding Halving Add
    168   def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic;
    169   def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic;
    170 
    171   // Vector Saturating Add
    172   def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic;
    173   def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic;
    174   def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic;
    175   def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic;
    176 
    177   // Vector Add High-Half
    178   // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
    179   // header is no longer supported.
    180   def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
    181 
    182   // Vector Rounding Add High-Half
    183   def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
    184 
    185   // Vector Saturating Doubling Multiply High
    186   def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic;
    187 
    188   // Vector Saturating Rounding Doubling Multiply High
    189   def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic;
    190 
    191   // Vector Polynominal Multiply
    192   def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic;
    193 
    194   // Vector Long Multiply
    195   def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic;
    196   def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic;
    197   def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic;
    198 
    199   // 64-bit polynomial multiply really returns an i128, which is not legal. Fake
    200   // it with a v16i8.
    201   def int_aarch64_neon_pmull64 :
    202         Intrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
    203 
    204   // Vector Extending Multiply
    205   def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic {
    206     let IntrProperties = [IntrNoMem, Commutative];
    207   }
    208 
    209   // Vector Saturating Doubling Long Multiply
    210   def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic;
    211   def int_aarch64_neon_sqdmulls_scalar
    212     : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
    213 
    214   // Vector Halving Subtract
    215   def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic;
    216   def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic;
    217 
    218   // Vector Saturating Subtract
    219   def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic;
    220   def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic;
    221 
    222   // Vector Subtract High-Half
    223   // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
    224   // header is no longer supported.
    225   def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
    226 
    227   // Vector Rounding Subtract High-Half
    228   def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
    229 
    230   // Vector Compare Absolute Greater-than-or-equal
    231   def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic;
    232 
    233   // Vector Compare Absolute Greater-than
    234   def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic;
    235 
    236   // Vector Absolute Difference
    237   def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic;
    238   def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic;
    239   def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic;
    240 
    241   // Scalar Absolute Difference
    242   def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic;
    243 
    244   // Vector Max
    245   def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic;
    246   def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic;
    247   def int_aarch64_neon_fmax : AdvSIMD_2VectorArg_Intrinsic;
    248   def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic;
    249 
    250   // Vector Max Across Lanes
    251   def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
    252   def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
    253   def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
    254   def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
    255 
    256   // Vector Min
    257   def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic;
    258   def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic;
    259   def int_aarch64_neon_fmin : AdvSIMD_2VectorArg_Intrinsic;
    260   def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic;
    261 
    262   // Vector Min/Max Number
    263   def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic;
    264   def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic;
    265 
    266   // Vector Min Across Lanes
    267   def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
    268   def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
    269   def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
    270   def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
    271 
    272   // Pairwise Add
    273   def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic;
    274 
    275   // Long Pairwise Add
    276   // FIXME: In theory, we shouldn't need intrinsics for saddlp or
    277   // uaddlp, but tblgen's type inference currently can't handle the
    278   // pattern fragments this ends up generating.
    279   def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
    280   def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
    281 
    282   // Folding Maximum
    283   def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic;
    284   def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic;
    285   def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic;
    286 
    287   // Folding Minimum
    288   def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic;
    289   def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic;
    290   def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic;
    291 
    292   // Reciprocal Estimate/Step
    293   def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic;
    294   def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic;
    295 
    296   // Reciprocal Exponent
    297   def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic;
    298 
    299   // Vector Saturating Shift Left
    300   def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic;
    301   def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic;
    302 
    303   // Vector Rounding Shift Left
    304   def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic;
    305   def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic;
    306 
    307   // Vector Saturating Rounding Shift Left
    308   def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic;
    309   def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic;
    310 
    311   // Vector Signed->Unsigned Shift Left by Constant
    312   def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic;
    313 
    314   // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant
    315   def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
    316 
    317   // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const
    318   def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
    319 
    320   // Vector Narrowing Shift Right by Constant
    321   def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
    322   def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
    323 
    324   // Vector Rounding Narrowing Shift Right by Constant
    325   def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
    326 
    327   // Vector Rounding Narrowing Saturating Shift Right by Constant
    328   def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
    329   def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
    330 
    331   // Vector Shift Left
    332   def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic;
    333   def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic;
    334 
    335   // Vector Widening Shift Left by Constant
    336   def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic;
    337   def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
    338   def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
    339 
    340   // Vector Shift Right by Constant and Insert
    341   def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic;
    342 
    343   // Vector Shift Left by Constant and Insert
    344   def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic;
    345 
    346   // Vector Saturating Narrow
    347   def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic;
    348   def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic;
    349   def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
    350   def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
    351 
    352   // Vector Saturating Extract and Unsigned Narrow
    353   def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic;
    354   def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic;
    355 
    356   // Vector Absolute Value
    357   def int_aarch64_neon_abs : AdvSIMD_1IntArg_Intrinsic;
    358 
    359   // Vector Saturating Absolute Value
    360   def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic;
    361 
    362   // Vector Saturating Negation
    363   def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic;
    364 
    365   // Vector Count Leading Sign Bits
    366   def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic;
    367 
    368   // Vector Reciprocal Estimate
    369   def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic;
    370   def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic;
    371 
    372   // Vector Square Root Estimate
    373   def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic;
    374   def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic;
    375 
    376   // Vector Bitwise Reverse
    377   def int_aarch64_neon_rbit : AdvSIMD_1VectorArg_Intrinsic;
    378 
    379   // Vector Conversions Between Half-Precision and Single-Precision.
    380   def int_aarch64_neon_vcvtfp2hf
    381     : Intrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
    382   def int_aarch64_neon_vcvthf2fp
    383     : Intrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>;
    384 
    385   // Vector Conversions Between Floating-point and Fixed-point.
    386   def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic;
    387   def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic;
    388   def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic;
    389   def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic;
    390 
    391   // Vector FP->Int Conversions
    392   def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic;
    393   def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic;
    394   def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic;
    395   def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic;
    396   def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic;
    397   def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic;
    398   def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic;
    399   def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic;
    400   def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic;
    401   def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic;
    402 
    403   // Vector FP Rounding: only ties to even is unrepresented by a normal
    404   // intrinsic.
    405   def int_aarch64_neon_frintn : AdvSIMD_1FloatArg_Intrinsic;
    406 
    407   // Scalar FP->Int conversions
    408 
    409   // Vector FP Inexact Narrowing
    410   def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic;
    411 
    412   // Scalar FP Inexact Narrowing
    413   def int_aarch64_sisd_fcvtxn : Intrinsic<[llvm_float_ty], [llvm_double_ty],
    414                                         [IntrNoMem]>;
    415 }
    416 
    417 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
    418   class AdvSIMD_2Vector2Index_Intrinsic
    419     : Intrinsic<[llvm_anyvector_ty],
    420                 [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty],
    421                 [IntrNoMem]>;
    422 }
    423 
    424 // Vector element to element moves
    425 def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic;
    426 
    427 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
    428   class AdvSIMD_1Vec_Load_Intrinsic
    429       : Intrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>],
    430                   [IntrReadMem, IntrArgMemOnly]>;
    431   class AdvSIMD_1Vec_Store_Lane_Intrinsic
    432     : Intrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty],
    433                 [IntrArgMemOnly, NoCapture<2>]>;
    434 
    435   class AdvSIMD_2Vec_Load_Intrinsic
    436     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
    437                 [LLVMAnyPointerType<LLVMMatchType<0>>],
    438                 [IntrReadMem, IntrArgMemOnly]>;
    439   class AdvSIMD_2Vec_Load_Lane_Intrinsic
    440     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
    441                 [LLVMMatchType<0>, LLVMMatchType<0>,
    442                  llvm_i64_ty, llvm_anyptr_ty],
    443                 [IntrReadMem, IntrArgMemOnly]>;
    444   class AdvSIMD_2Vec_Store_Intrinsic
    445     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
    446                      LLVMAnyPointerType<LLVMMatchType<0>>],
    447                 [IntrArgMemOnly, NoCapture<2>]>;
    448   class AdvSIMD_2Vec_Store_Lane_Intrinsic
    449     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
    450                  llvm_i64_ty, llvm_anyptr_ty],
    451                 [IntrArgMemOnly, NoCapture<3>]>;
    452 
    453   class AdvSIMD_3Vec_Load_Intrinsic
    454     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>],
    455                 [LLVMAnyPointerType<LLVMMatchType<0>>],
    456                 [IntrReadMem, IntrArgMemOnly]>;
    457   class AdvSIMD_3Vec_Load_Lane_Intrinsic
    458     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>],
    459                 [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>,
    460                  llvm_i64_ty, llvm_anyptr_ty],
    461                 [IntrReadMem, IntrArgMemOnly]>;
    462   class AdvSIMD_3Vec_Store_Intrinsic
    463     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
    464                      LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>],
    465                 [IntrArgMemOnly, NoCapture<3>]>;
    466   class AdvSIMD_3Vec_Store_Lane_Intrinsic
    467     : Intrinsic<[], [llvm_anyvector_ty,
    468                  LLVMMatchType<0>, LLVMMatchType<0>,
    469                  llvm_i64_ty, llvm_anyptr_ty],
    470                 [IntrArgMemOnly, NoCapture<4>]>;
    471 
    472   class AdvSIMD_4Vec_Load_Intrinsic
    473     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>,
    474                  LLVMMatchType<0>, LLVMMatchType<0>],
    475                 [LLVMAnyPointerType<LLVMMatchType<0>>],
    476                 [IntrReadMem, IntrArgMemOnly]>;
    477   class AdvSIMD_4Vec_Load_Lane_Intrinsic
    478     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>,
    479                  LLVMMatchType<0>, LLVMMatchType<0>],
    480                 [LLVMMatchType<0>, LLVMMatchType<0>,
    481                  LLVMMatchType<0>, LLVMMatchType<0>,
    482                  llvm_i64_ty, llvm_anyptr_ty],
    483                 [IntrReadMem, IntrArgMemOnly]>;
    484   class AdvSIMD_4Vec_Store_Intrinsic
    485     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
    486                  LLVMMatchType<0>, LLVMMatchType<0>,
    487                  LLVMAnyPointerType<LLVMMatchType<0>>],
    488                 [IntrArgMemOnly, NoCapture<4>]>;
    489   class AdvSIMD_4Vec_Store_Lane_Intrinsic
    490     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
    491                  LLVMMatchType<0>, LLVMMatchType<0>,
    492                  llvm_i64_ty, llvm_anyptr_ty],
    493                 [IntrArgMemOnly, NoCapture<5>]>;
    494 }
    495 
    496 // Memory ops
    497 
    498 def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic;
    499 def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic;
    500 def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic;
    501 
    502 def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic;
    503 def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic;
    504 def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic;
    505 
    506 def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic;
    507 def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic;
    508 def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic;
    509 
    510 def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic;
    511 def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic;
    512 def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic;
    513 
    514 def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic;
    515 def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic;
    516 def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic;
    517 
    518 def int_aarch64_neon_st2  : AdvSIMD_2Vec_Store_Intrinsic;
    519 def int_aarch64_neon_st3  : AdvSIMD_3Vec_Store_Intrinsic;
    520 def int_aarch64_neon_st4  : AdvSIMD_4Vec_Store_Intrinsic;
    521 
    522 def int_aarch64_neon_st2lane  : AdvSIMD_2Vec_Store_Lane_Intrinsic;
    523 def int_aarch64_neon_st3lane  : AdvSIMD_3Vec_Store_Lane_Intrinsic;
    524 def int_aarch64_neon_st4lane  : AdvSIMD_4Vec_Store_Lane_Intrinsic;
    525 
    526 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
    527   class AdvSIMD_Tbl1_Intrinsic
    528     : Intrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>],
    529                 [IntrNoMem]>;
    530   class AdvSIMD_Tbl2_Intrinsic
    531     : Intrinsic<[llvm_anyvector_ty],
    532                 [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>;
    533   class AdvSIMD_Tbl3_Intrinsic
    534     : Intrinsic<[llvm_anyvector_ty],
    535                 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
    536                  LLVMMatchType<0>],
    537                 [IntrNoMem]>;
    538   class AdvSIMD_Tbl4_Intrinsic
    539     : Intrinsic<[llvm_anyvector_ty],
    540                 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
    541                  LLVMMatchType<0>],
    542                 [IntrNoMem]>;
    543 
    544   class AdvSIMD_Tbx1_Intrinsic
    545     : Intrinsic<[llvm_anyvector_ty],
    546                 [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>],
    547                 [IntrNoMem]>;
    548   class AdvSIMD_Tbx2_Intrinsic
    549     : Intrinsic<[llvm_anyvector_ty],
    550                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
    551                  LLVMMatchType<0>],
    552                 [IntrNoMem]>;
    553   class AdvSIMD_Tbx3_Intrinsic
    554     : Intrinsic<[llvm_anyvector_ty],
    555                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
    556                  llvm_v16i8_ty, LLVMMatchType<0>],
    557                 [IntrNoMem]>;
    558   class AdvSIMD_Tbx4_Intrinsic
    559     : Intrinsic<[llvm_anyvector_ty],
    560                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
    561                  llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>],
    562                 [IntrNoMem]>;
    563 }
    564 def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic;
    565 def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic;
    566 def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic;
    567 def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic;
    568 
    569 def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic;
    570 def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic;
    571 def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic;
    572 def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic;
    573 
    574 let TargetPrefix = "aarch64" in {
    575   class Crypto_AES_DataKey_Intrinsic
    576     : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
    577 
    578   class Crypto_AES_Data_Intrinsic
    579     : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>;
    580 
    581   // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
    582   // (v4i32).
    583   class Crypto_SHA_5Hash4Schedule_Intrinsic
    584     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty],
    585                 [IntrNoMem]>;
    586 
    587   // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
    588   // (v4i32).
    589   class Crypto_SHA_1Hash_Intrinsic
    590     : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
    591 
    592   // SHA intrinsic taking 8 words of the schedule
    593   class Crypto_SHA_8Schedule_Intrinsic
    594     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
    595 
    596   // SHA intrinsic taking 12 words of the schedule
    597   class Crypto_SHA_12Schedule_Intrinsic
    598     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
    599                 [IntrNoMem]>;
    600 
    601   // SHA intrinsic taking 8 words of the hash and 4 of the schedule.
    602   class Crypto_SHA_8Hash4Schedule_Intrinsic
    603     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
    604                 [IntrNoMem]>;
    605 }
    606 
    607 // AES
    608 def int_aarch64_crypto_aese   : Crypto_AES_DataKey_Intrinsic;
    609 def int_aarch64_crypto_aesd   : Crypto_AES_DataKey_Intrinsic;
    610 def int_aarch64_crypto_aesmc  : Crypto_AES_Data_Intrinsic;
    611 def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic;
    612 
    613 // SHA1
    614 def int_aarch64_crypto_sha1c  : Crypto_SHA_5Hash4Schedule_Intrinsic;
    615 def int_aarch64_crypto_sha1p  : Crypto_SHA_5Hash4Schedule_Intrinsic;
    616 def int_aarch64_crypto_sha1m  : Crypto_SHA_5Hash4Schedule_Intrinsic;
    617 def int_aarch64_crypto_sha1h  : Crypto_SHA_1Hash_Intrinsic;
    618 
    619 def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic;
    620 def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic;
    621 
    622 // SHA256
    623 def int_aarch64_crypto_sha256h   : Crypto_SHA_8Hash4Schedule_Intrinsic;
    624 def int_aarch64_crypto_sha256h2  : Crypto_SHA_8Hash4Schedule_Intrinsic;
    625 def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic;
    626 def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic;
    627 
    628 //===----------------------------------------------------------------------===//
    629 // CRC32
    630 
    631 let TargetPrefix = "aarch64" in {
    632 
    633 def int_aarch64_crc32b  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
    634     [IntrNoMem]>;
    635 def int_aarch64_crc32cb : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
    636     [IntrNoMem]>;
    637 def int_aarch64_crc32h  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
    638     [IntrNoMem]>;
    639 def int_aarch64_crc32ch : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
    640     [IntrNoMem]>;
    641 def int_aarch64_crc32w  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
    642     [IntrNoMem]>;
    643 def int_aarch64_crc32cw : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
    644     [IntrNoMem]>;
    645 def int_aarch64_crc32x  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
    646     [IntrNoMem]>;
    647 def int_aarch64_crc32cx : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
    648     [IntrNoMem]>;
    649 }
    650