Home | History | Annotate | Download | only in X86
      1 // Bitcasts between 512-bit vector types. Return the original type since
      2 // no instruction is needed for the conversion
      3 let Predicates = [HasAVX512] in {
      4   def : Pat<(v8f64  (bitconvert (v16f32 VR512:$src))), (v8f64 VR512:$src)>;
      5   def : Pat<(v8f64  (bitconvert (v16i32 VR512:$src))), (v8f64 VR512:$src)>;
      6   def : Pat<(v8f64  (bitconvert (v8i64 VR512:$src))),  (v8f64 VR512:$src)>;
      7   def : Pat<(v16f32 (bitconvert (v16i32 VR512:$src))), (v16f32 VR512:$src)>;
      8   def : Pat<(v16f32 (bitconvert (v8i64 VR512:$src))),  (v16f32 VR512:$src)>;
      9   def : Pat<(v16f32 (bitconvert (v8f64 VR512:$src))),  (v16f32 VR512:$src)>;
     10   def : Pat<(v8i64  (bitconvert (v16f32 VR512:$src))), (v8i64 VR512:$src)>;
     11   def : Pat<(v8i64  (bitconvert (v16i32 VR512:$src))), (v8i64 VR512:$src)>;
     12   def : Pat<(v8i64  (bitconvert (v8f64 VR512:$src))),  (v8i64 VR512:$src)>;
     13   def : Pat<(v16i32 (bitconvert (v16f32 VR512:$src))), (v16i32 VR512:$src)>;
     14   def : Pat<(v16i32 (bitconvert (v8i64 VR512:$src))),  (v16i32 VR512:$src)>;
     15   def : Pat<(v16i32 (bitconvert (v8f64 VR512:$src))),  (v16i32 VR512:$src)>;
     16   def : Pat<(v8f64  (bitconvert (v8i64 VR512:$src))),  (v8f64 VR512:$src)>;
     17 
     18   def : Pat<(v2i64 (bitconvert (v4i32 VR128X:$src))), (v2i64 VR128X:$src)>;
     19   def : Pat<(v2i64 (bitconvert (v8i16 VR128X:$src))), (v2i64 VR128X:$src)>;
     20   def : Pat<(v2i64 (bitconvert (v16i8 VR128X:$src))), (v2i64 VR128X:$src)>;
     21   def : Pat<(v2i64 (bitconvert (v2f64 VR128X:$src))), (v2i64 VR128X:$src)>;
     22   def : Pat<(v2i64 (bitconvert (v4f32 VR128X:$src))), (v2i64 VR128X:$src)>;
     23   def : Pat<(v4i32 (bitconvert (v2i64 VR128X:$src))), (v4i32 VR128X:$src)>;
     24   def : Pat<(v4i32 (bitconvert (v8i16 VR128X:$src))), (v4i32 VR128X:$src)>;
     25   def : Pat<(v4i32 (bitconvert (v16i8 VR128X:$src))), (v4i32 VR128X:$src)>;
     26   def : Pat<(v4i32 (bitconvert (v2f64 VR128X:$src))), (v4i32 VR128X:$src)>;
     27   def : Pat<(v4i32 (bitconvert (v4f32 VR128X:$src))), (v4i32 VR128X:$src)>;
     28   def : Pat<(v8i16 (bitconvert (v2i64 VR128X:$src))), (v8i16 VR128X:$src)>;
     29   def : Pat<(v8i16 (bitconvert (v4i32 VR128X:$src))), (v8i16 VR128X:$src)>;
     30   def : Pat<(v8i16 (bitconvert (v16i8 VR128X:$src))), (v8i16 VR128X:$src)>;
     31   def : Pat<(v8i16 (bitconvert (v2f64 VR128X:$src))), (v8i16 VR128X:$src)>;
     32   def : Pat<(v8i16 (bitconvert (v4f32 VR128X:$src))), (v8i16 VR128X:$src)>;
     33   def : Pat<(v16i8 (bitconvert (v2i64 VR128X:$src))), (v16i8 VR128X:$src)>;
     34   def : Pat<(v16i8 (bitconvert (v4i32 VR128X:$src))), (v16i8 VR128X:$src)>;
     35   def : Pat<(v16i8 (bitconvert (v8i16 VR128X:$src))), (v16i8 VR128X:$src)>;
     36   def : Pat<(v16i8 (bitconvert (v2f64 VR128X:$src))), (v16i8 VR128X:$src)>;
     37   def : Pat<(v16i8 (bitconvert (v4f32 VR128X:$src))), (v16i8 VR128X:$src)>;
     38   def : Pat<(v4f32 (bitconvert (v2i64 VR128X:$src))), (v4f32 VR128X:$src)>;
     39   def : Pat<(v4f32 (bitconvert (v4i32 VR128X:$src))), (v4f32 VR128X:$src)>;
     40   def : Pat<(v4f32 (bitconvert (v8i16 VR128X:$src))), (v4f32 VR128X:$src)>;
     41   def : Pat<(v4f32 (bitconvert (v16i8 VR128X:$src))), (v4f32 VR128X:$src)>;
     42   def : Pat<(v4f32 (bitconvert (v2f64 VR128X:$src))), (v4f32 VR128X:$src)>;
     43   def : Pat<(v2f64 (bitconvert (v2i64 VR128X:$src))), (v2f64 VR128X:$src)>;
     44   def : Pat<(v2f64 (bitconvert (v4i32 VR128X:$src))), (v2f64 VR128X:$src)>;
     45   def : Pat<(v2f64 (bitconvert (v8i16 VR128X:$src))), (v2f64 VR128X:$src)>;
     46   def : Pat<(v2f64 (bitconvert (v16i8 VR128X:$src))), (v2f64 VR128X:$src)>;
     47   def : Pat<(v2f64 (bitconvert (v4f32 VR128X:$src))), (v2f64 VR128X:$src)>;
     48 
     49 // Bitcasts between 256-bit vector types. Return the original type since
     50 // no instruction is needed for the conversion
     51   def : Pat<(v4f64  (bitconvert (v8f32 VR256X:$src))),  (v4f64 VR256X:$src)>;
     52   def : Pat<(v4f64  (bitconvert (v8i32 VR256X:$src))),  (v4f64 VR256X:$src)>;
     53   def : Pat<(v4f64  (bitconvert (v4i64 VR256X:$src))),  (v4f64 VR256X:$src)>;
     54   def : Pat<(v4f64  (bitconvert (v16i16 VR256X:$src))), (v4f64 VR256X:$src)>;
     55   def : Pat<(v4f64  (bitconvert (v32i8 VR256X:$src))),  (v4f64 VR256X:$src)>;
     56   def : Pat<(v8f32  (bitconvert (v8i32 VR256X:$src))),  (v8f32 VR256X:$src)>;
     57   def : Pat<(v8f32  (bitconvert (v4i64 VR256X:$src))),  (v8f32 VR256X:$src)>;
     58   def : Pat<(v8f32  (bitconvert (v4f64 VR256X:$src))),  (v8f32 VR256X:$src)>;
     59   def : Pat<(v8f32  (bitconvert (v32i8 VR256X:$src))),  (v8f32 VR256X:$src)>;
     60   def : Pat<(v8f32  (bitconvert (v16i16 VR256X:$src))), (v8f32 VR256X:$src)>;
     61   def : Pat<(v4i64  (bitconvert (v8f32 VR256X:$src))),  (v4i64 VR256X:$src)>;
     62   def : Pat<(v4i64  (bitconvert (v8i32 VR256X:$src))),  (v4i64 VR256X:$src)>;
     63   def : Pat<(v4i64  (bitconvert (v4f64 VR256X:$src))),  (v4i64 VR256X:$src)>;
     64   def : Pat<(v4i64  (bitconvert (v32i8 VR256X:$src))),  (v4i64 VR256X:$src)>;
     65   def : Pat<(v4i64  (bitconvert (v16i16 VR256X:$src))), (v4i64 VR256X:$src)>;
     66   def : Pat<(v32i8  (bitconvert (v4f64 VR256X:$src))),  (v32i8 VR256X:$src)>;
     67   def : Pat<(v32i8  (bitconvert (v4i64 VR256X:$src))),  (v32i8 VR256X:$src)>;
     68   def : Pat<(v32i8  (bitconvert (v8f32 VR256X:$src))),  (v32i8 VR256X:$src)>;
     69   def : Pat<(v32i8  (bitconvert (v8i32 VR256X:$src))),  (v32i8 VR256X:$src)>;
     70   def : Pat<(v32i8  (bitconvert (v16i16 VR256X:$src))), (v32i8 VR256X:$src)>;
     71   def : Pat<(v8i32  (bitconvert (v32i8 VR256X:$src))),  (v8i32 VR256X:$src)>;
     72   def : Pat<(v8i32  (bitconvert (v16i16 VR256X:$src))), (v8i32 VR256X:$src)>;
     73   def : Pat<(v8i32  (bitconvert (v8f32 VR256X:$src))),  (v8i32 VR256X:$src)>;
     74   def : Pat<(v8i32  (bitconvert (v4i64 VR256X:$src))),  (v8i32 VR256X:$src)>;
     75   def : Pat<(v8i32  (bitconvert (v4f64 VR256X:$src))),  (v8i32 VR256X:$src)>;
     76   def : Pat<(v16i16 (bitconvert (v8f32 VR256X:$src))),  (v16i16 VR256X:$src)>;
     77   def : Pat<(v16i16 (bitconvert (v8i32 VR256X:$src))),  (v16i16 VR256X:$src)>;
     78   def : Pat<(v16i16 (bitconvert (v4i64 VR256X:$src))),  (v16i16 VR256X:$src)>;
     79   def : Pat<(v16i16 (bitconvert (v4f64 VR256X:$src))),  (v16i16 VR256X:$src)>;
     80   def : Pat<(v16i16 (bitconvert (v32i8 VR256X:$src))),  (v16i16 VR256X:$src)>;
     81 }
     82 
     83 //
     84 // AVX-512: VPXOR instruction writes zero to its upper part, it's safe build zeros.
     85 //
     86 
     87 let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
     88     isPseudo = 1, Predicates = [HasAVX512] in {
     89 def AVX512_512_SET0 : I<0, Pseudo, (outs VR512:$dst), (ins), "",
     90                [(set VR512:$dst, (v16f32 immAllZerosV))]>;
     91 }
     92 
     93 let Predicates = [HasAVX512] in {
     94 def : Pat<(v8i64 immAllZerosV), (AVX512_512_SET0)>;
     95 def : Pat<(v16i32 immAllZerosV), (AVX512_512_SET0)>;
     96 def : Pat<(v8f64 immAllZerosV), (AVX512_512_SET0)>;
     97 }
     98 
     99 //===----------------------------------------------------------------------===//
    100 // AVX-512 - VECTOR INSERT
    101 //
    102 // -- 32x8 form --
    103 let hasSideEffects = 0, ExeDomain = SSEPackedSingle in {
    104 def VINSERTF32x4rr : AVX512AIi8<0x18, MRMSrcReg, (outs VR512:$dst),
    105           (ins VR512:$src1, VR128X:$src2, i8imm:$src3),
    106           "vinsertf32x4\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
    107           []>, EVEX_4V, EVEX_V512;
    108 let mayLoad = 1 in
    109 def VINSERTF32x4rm : AVX512AIi8<0x18, MRMSrcMem, (outs VR512:$dst),
    110           (ins VR512:$src1, f128mem:$src2, i8imm:$src3),
    111           "vinsertf32x4\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
    112           []>, EVEX_4V, EVEX_V512, EVEX_CD8<32, CD8VT4>;
    113 }
    114 
    115 // -- 64x4 fp form --
    116 let hasSideEffects = 0, ExeDomain = SSEPackedDouble in {
    117 def VINSERTF64x4rr : AVX512AIi8<0x1a, MRMSrcReg, (outs VR512:$dst),
    118           (ins VR512:$src1, VR256X:$src2, i8imm:$src3),
    119           "vinsertf64x4\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
    120           []>, EVEX_4V, EVEX_V512, VEX_W;
    121 let mayLoad = 1 in
    122 def VINSERTF64x4rm : AVX512AIi8<0x1a, MRMSrcMem, (outs VR512:$dst),
    123           (ins VR512:$src1, i256mem:$src2, i8imm:$src3),
    124           "vinsertf64x4\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
    125           []>, EVEX_4V, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT4>;
    126 }
    127 // -- 32x4 integer form --
    128 let hasSideEffects = 0 in {
    129 def VINSERTI32x4rr : AVX512AIi8<0x38, MRMSrcReg, (outs VR512:$dst),
    130           (ins VR512:$src1, VR128X:$src2, i8imm:$src3),
    131           "vinserti32x4\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
    132           []>, EVEX_4V, EVEX_V512;
    133 let mayLoad = 1 in
    134 def VINSERTI32x4rm : AVX512AIi8<0x38, MRMSrcMem, (outs VR512:$dst),
    135           (ins VR512:$src1, i128mem:$src2, i8imm:$src3),
    136           "vinserti32x4\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
    137           []>, EVEX_4V, EVEX_V512, EVEX_CD8<32, CD8VT4>;
    138 
    139 }
    140 
    141 let hasSideEffects = 0 in {
    142 // -- 64x4 form --
    143 def VINSERTI64x4rr : AVX512AIi8<0x3a, MRMSrcReg, (outs VR512:$dst),
    144           (ins VR512:$src1, VR256X:$src2, i8imm:$src3),
    145           "vinserti64x4\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
    146           []>, EVEX_4V, EVEX_V512, VEX_W;
    147 let mayLoad = 1 in
    148 def VINSERTI64x4rm : AVX512AIi8<0x3a, MRMSrcMem, (outs VR512:$dst),
    149           (ins VR512:$src1, i256mem:$src2, i8imm:$src3),
    150           "vinserti64x4\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
    151           []>, EVEX_4V, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT4>;
    152 }
    153 
    154 def : Pat<(vinsert128_insert:$ins (v16f32 VR512:$src1), (v4f32 VR128X:$src2),
    155            (iPTR imm)), (VINSERTF32x4rr VR512:$src1, VR128X:$src2,
    156                         (INSERT_get_vinsert128_imm VR512:$ins))>;
    157 def : Pat<(vinsert128_insert:$ins (v8f64  VR512:$src1), (v2f64 VR128X:$src2),
    158            (iPTR imm)), (VINSERTF32x4rr VR512:$src1, VR128X:$src2,
    159                         (INSERT_get_vinsert128_imm VR512:$ins))>;
    160 def : Pat<(vinsert128_insert:$ins (v8i64  VR512:$src1), (v2i64 VR128X:$src2),
    161            (iPTR imm)), (VINSERTI32x4rr VR512:$src1, VR128X:$src2,
    162                         (INSERT_get_vinsert128_imm VR512:$ins))>;
    163 def : Pat<(vinsert128_insert:$ins (v16i32 VR512:$src1), (v4i32 VR128X:$src2),
    164            (iPTR imm)), (VINSERTI32x4rr VR512:$src1, VR128X:$src2,
    165                         (INSERT_get_vinsert128_imm VR512:$ins))>;
    166 
    167 def : Pat<(vinsert128_insert:$ins (v16f32 VR512:$src1), (loadv4f32 addr:$src2),
    168            (iPTR imm)), (VINSERTF32x4rm VR512:$src1, addr:$src2,
    169                         (INSERT_get_vinsert128_imm VR512:$ins))>;
    170 def : Pat<(vinsert128_insert:$ins (v16i32 VR512:$src1),
    171                   (bc_v4i32 (loadv2i64 addr:$src2)),
    172            (iPTR imm)), (VINSERTI32x4rm VR512:$src1, addr:$src2,
    173                         (INSERT_get_vinsert128_imm VR512:$ins))>;
    174 def : Pat<(vinsert128_insert:$ins (v8f64  VR512:$src1), (loadv2f64 addr:$src2),
    175            (iPTR imm)), (VINSERTF32x4rm VR512:$src1, addr:$src2,
    176                         (INSERT_get_vinsert128_imm VR512:$ins))>;
    177 def : Pat<(vinsert128_insert:$ins (v8i64  VR512:$src1), (loadv2i64 addr:$src2),
    178            (iPTR imm)), (VINSERTI32x4rm VR512:$src1, addr:$src2,
    179                         (INSERT_get_vinsert128_imm VR512:$ins))>;
    180 
    181 def : Pat<(vinsert256_insert:$ins (v16f32  VR512:$src1), (v8f32 VR256X:$src2),
    182            (iPTR imm)), (VINSERTF64x4rr VR512:$src1, VR256X:$src2,
    183                         (INSERT_get_vinsert256_imm VR512:$ins))>;
    184 def : Pat<(vinsert256_insert:$ins (v8f64  VR512:$src1), (v4f64 VR256X:$src2),
    185            (iPTR imm)), (VINSERTF64x4rr VR512:$src1, VR256X:$src2,
    186                         (INSERT_get_vinsert256_imm VR512:$ins))>;
    187 def : Pat<(vinsert128_insert:$ins (v8i64  VR512:$src1), (v4i64 VR256X:$src2),
    188            (iPTR imm)), (VINSERTI64x4rr VR512:$src1, VR256X:$src2,
    189                         (INSERT_get_vinsert256_imm VR512:$ins))>;
    190 def : Pat<(vinsert128_insert:$ins (v16i32 VR512:$src1), (v8i32 VR256X:$src2),
    191            (iPTR imm)), (VINSERTI64x4rr VR512:$src1, VR256X:$src2,
    192                         (INSERT_get_vinsert256_imm VR512:$ins))>;
    193 
    194 def : Pat<(vinsert256_insert:$ins (v16f32  VR512:$src1), (loadv8f32 addr:$src2),
    195            (iPTR imm)), (VINSERTF64x4rm VR512:$src1, addr:$src2,
    196                         (INSERT_get_vinsert256_imm VR512:$ins))>;
    197 def : Pat<(vinsert256_insert:$ins (v8f64  VR512:$src1), (loadv4f64 addr:$src2),
    198            (iPTR imm)), (VINSERTF64x4rm VR512:$src1, addr:$src2,
    199                         (INSERT_get_vinsert256_imm VR512:$ins))>;
    200 def : Pat<(vinsert256_insert:$ins (v8i64  VR512:$src1), (loadv4i64 addr:$src2),
    201            (iPTR imm)), (VINSERTI64x4rm VR512:$src1, addr:$src2,
    202                         (INSERT_get_vinsert256_imm VR512:$ins))>;
    203 def : Pat<(vinsert256_insert:$ins (v16i32 VR512:$src1),
    204 	                (bc_v8i32 (loadv4i64 addr:$src2)),
    205            (iPTR imm)), (VINSERTI64x4rm VR512:$src1, addr:$src2,
    206                         (INSERT_get_vinsert256_imm VR512:$ins))>;
    207 
    208 // vinsertps - insert f32 to XMM
    209 def VINSERTPSzrr : AVX512AIi8<0x21, MRMSrcReg, (outs VR128X:$dst),
    210       (ins VR128X:$src1, VR128X:$src2, u32u8imm:$src3),
    211       "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
    212       [(set VR128X:$dst, (X86insertps VR128X:$src1, VR128X:$src2, imm:$src3))]>,
    213       EVEX_4V;
    214 def VINSERTPSzrm: AVX512AIi8<0x21, MRMSrcMem, (outs VR128X:$dst),
    215       (ins VR128X:$src1, f32mem:$src2, u32u8imm:$src3),
    216       "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
    217       [(set VR128X:$dst, (X86insertps VR128X:$src1,
    218                           (v4f32 (scalar_to_vector (loadf32 addr:$src2))),
    219                           imm:$src3))]>, EVEX_4V, EVEX_CD8<32, CD8VT1>;
    220 
    221 //===----------------------------------------------------------------------===//
    222 // AVX-512 VECTOR EXTRACT
    223 //---
    224 let hasSideEffects = 0, ExeDomain = SSEPackedSingle in {
    225 // -- 32x4 form --
    226 def VEXTRACTF32x4rr : AVX512AIi8<0x19, MRMDestReg, (outs VR128X:$dst),
    227           (ins VR512:$src1, i8imm:$src2),
    228           "vextractf32x4\t{$src2, $src1, $dst|$dst, $src1, $src2}",
    229           []>, EVEX, EVEX_V512;
    230 def VEXTRACTF32x4mr : AVX512AIi8<0x19, MRMDestMem, (outs),
    231           (ins f128mem:$dst, VR512:$src1, i8imm:$src2),
    232           "vextractf32x4\t{$src2, $src1, $dst|$dst, $src1, $src2}",
    233           []>, EVEX, EVEX_V512, EVEX_CD8<32, CD8VT4>;
    234 
    235 // -- 64x4 form --
    236 def VEXTRACTF64x4rr : AVX512AIi8<0x1b, MRMDestReg, (outs VR256X:$dst),
    237           (ins VR512:$src1, i8imm:$src2),
    238           "vextractf64x4\t{$src2, $src1, $dst|$dst, $src1, $src2}",
    239           []>, EVEX, EVEX_V512, VEX_W;
    240 let mayStore = 1 in
    241 def VEXTRACTF64x4mr : AVX512AIi8<0x1b, MRMDestMem, (outs),
    242           (ins f256mem:$dst, VR512:$src1, i8imm:$src2),
    243           "vextractf64x4\t{$src2, $src1, $dst|$dst, $src1, $src2}",
    244           []>, EVEX, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT4>;
    245 }
    246 
    247 let hasSideEffects = 0 in {
    248 // -- 32x4 form --
    249 def VEXTRACTI32x4rr : AVX512AIi8<0x39, MRMDestReg, (outs VR128X:$dst),
    250           (ins VR512:$src1, i8imm:$src2),
    251           "vextracti32x4\t{$src2, $src1, $dst|$dst, $src1, $src2}",
    252           []>, EVEX, EVEX_V512;
    253 def VEXTRACTI32x4mr : AVX512AIi8<0x39, MRMDestMem, (outs),
    254           (ins i128mem:$dst, VR512:$src1, i8imm:$src2),
    255           "vextracti32x4\t{$src2, $src1, $dst|$dst, $src1, $src2}",
    256           []>, EVEX, EVEX_V512, EVEX_CD8<32, CD8VT4>;
    257 
    258 // -- 64x4 form --
    259 def VEXTRACTI64x4rr : AVX512AIi8<0x3b, MRMDestReg, (outs VR256X:$dst),
    260           (ins VR512:$src1, i8imm:$src2),
    261           "vextracti64x4\t{$src2, $src1, $dst|$dst, $src1, $src2}",
    262           []>, EVEX, EVEX_V512, VEX_W;
    263 let mayStore = 1 in
    264 def VEXTRACTI64x4mr : AVX512AIi8<0x3b, MRMDestMem, (outs),
    265           (ins i256mem:$dst, VR512:$src1, i8imm:$src2),
    266           "vextracti64x4\t{$src2, $src1, $dst|$dst, $src1, $src2}",
    267           []>, EVEX, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT4>;
    268 }
    269 
    270 def : Pat<(vextract128_extract:$ext (v16f32 VR512:$src1), (iPTR imm)),
    271           (v4f32 (VEXTRACTF32x4rr VR512:$src1,
    272                   (EXTRACT_get_vextract128_imm VR128X:$ext)))>;
    273 
    274 def : Pat<(vextract128_extract:$ext VR512:$src1, (iPTR imm)),
    275           (v4i32 (VEXTRACTF32x4rr VR512:$src1,
    276                   (EXTRACT_get_vextract128_imm VR128X:$ext)))>;
    277 
    278 def : Pat<(vextract128_extract:$ext (v8f64 VR512:$src1), (iPTR imm)),
    279           (v2f64 (VEXTRACTF32x4rr VR512:$src1,
    280                   (EXTRACT_get_vextract128_imm VR128X:$ext)))>;
    281 
    282 def : Pat<(vextract128_extract:$ext (v8i64 VR512:$src1), (iPTR imm)),
    283           (v2i64 (VEXTRACTI32x4rr VR512:$src1,
    284                   (EXTRACT_get_vextract128_imm VR128X:$ext)))>;
    285 
    286 
    287 def : Pat<(vextract256_extract:$ext (v16f32 VR512:$src1), (iPTR imm)),
    288           (v8f32 (VEXTRACTF64x4rr VR512:$src1,
    289                   (EXTRACT_get_vextract256_imm VR256X:$ext)))>;
    290 
    291 def : Pat<(vextract256_extract:$ext (v16i32 VR512:$src1), (iPTR imm)),
    292           (v8i32 (VEXTRACTI64x4rr VR512:$src1,
    293                     (EXTRACT_get_vextract256_imm VR256X:$ext)))>;
    294 
    295 def : Pat<(vextract256_extract:$ext (v8f64 VR512:$src1), (iPTR imm)),
    296           (v4f64 (VEXTRACTF64x4rr VR512:$src1,
    297                   (EXTRACT_get_vextract256_imm VR256X:$ext)))>;
    298 
    299 def : Pat<(vextract256_extract:$ext (v8i64 VR512:$src1), (iPTR imm)),
    300           (v4i64 (VEXTRACTI64x4rr VR512:$src1,
    301                   (EXTRACT_get_vextract256_imm VR256X:$ext)))>;
    302 
    303 // A 256-bit subvector extract from the first 512-bit vector position
    304 // is a subregister copy that needs no instruction.
    305 def : Pat<(v8i32 (extract_subvector (v16i32 VR512:$src), (iPTR 0))),
    306           (v8i32 (EXTRACT_SUBREG (v16i32 VR512:$src), sub_ymm))>;
    307 def : Pat<(v8f32 (extract_subvector (v16f32 VR512:$src), (iPTR 0))),
    308           (v8f32 (EXTRACT_SUBREG (v16f32 VR512:$src), sub_ymm))>;
    309 def : Pat<(v4i64 (extract_subvector (v8i64 VR512:$src), (iPTR 0))),
    310           (v4i64 (EXTRACT_SUBREG (v8i64 VR512:$src), sub_ymm))>;
    311 def : Pat<(v4f64 (extract_subvector (v8f64 VR512:$src), (iPTR 0))),
    312           (v4f64 (EXTRACT_SUBREG (v8f64 VR512:$src), sub_ymm))>;
    313 
    314 // zmm -> xmm
    315 def : Pat<(v4i32 (extract_subvector (v16i32 VR512:$src), (iPTR 0))),
    316           (v4i32 (EXTRACT_SUBREG (v16i32 VR512:$src), sub_xmm))>;
    317 def : Pat<(v2i64 (extract_subvector (v8i64 VR512:$src), (iPTR 0))),
    318           (v2i64 (EXTRACT_SUBREG (v8i64 VR512:$src), sub_xmm))>;
    319 def : Pat<(v2f64 (extract_subvector (v8f64 VR512:$src), (iPTR 0))),
    320           (v2f64 (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm))>;
    321 def : Pat<(v4f32 (extract_subvector (v16f32 VR512:$src), (iPTR 0))),
    322           (v4f32 (EXTRACT_SUBREG (v16f32 VR512:$src), sub_xmm))>;
    323 
    324 
    325 // A 128-bit subvector insert to the first 512-bit vector position
    326 // is a subregister copy that needs no instruction.
    327 def : Pat<(insert_subvector undef, (v2i64 VR128X:$src), (iPTR 0)),
    328           (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)),
    329           (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
    330           sub_ymm)>;
    331 def : Pat<(insert_subvector undef, (v2f64 VR128X:$src), (iPTR 0)),
    332           (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)),
    333           (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
    334           sub_ymm)>;
    335 def : Pat<(insert_subvector undef, (v4i32 VR128X:$src), (iPTR 0)),
    336           (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)),
    337           (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
    338           sub_ymm)>;
    339 def : Pat<(insert_subvector undef, (v4f32 VR128X:$src), (iPTR 0)),
    340           (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)),
    341           (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
    342           sub_ymm)>;
    343 
    344 def : Pat<(insert_subvector undef, (v4i64 VR256X:$src), (iPTR 0)),
    345           (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>;
    346 def : Pat<(insert_subvector undef, (v4f64 VR256X:$src), (iPTR 0)),
    347           (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>;
    348 def : Pat<(insert_subvector undef, (v8i32 VR256X:$src), (iPTR 0)),
    349           (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>;
    350 def : Pat<(insert_subvector undef, (v8f32 VR256X:$src), (iPTR 0)),
    351           (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>;
    352 
    353 // vextractps - extract 32 bits from XMM
    354 def VEXTRACTPSzrr : AVX512AIi8<0x17, MRMDestReg, (outs GR32:$dst),
    355       (ins VR128X:$src1, u32u8imm:$src2),
    356       "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
    357       [(set GR32:$dst, (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2))]>,
    358       EVEX;
    359 
    360 def VEXTRACTPSzmr : AVX512AIi8<0x17, MRMDestMem, (outs),
    361       (ins f32mem:$dst, VR128X:$src1, u32u8imm:$src2),
    362       "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
    363       [(store (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2),
    364                           addr:$dst)]>, EVEX, EVEX_CD8<32, CD8VT1>;
    365 
    366 //===---------------------------------------------------------------------===//
    367 // AVX-512 BROADCAST
    368 //---
    369 multiclass avx512_fp_broadcast<bits<8> opc, string OpcodeStr, 
    370                          RegisterClass DestRC,
    371                          RegisterClass SrcRC, X86MemOperand x86memop> {
    372   def rr : AVX5128I<opc, MRMSrcReg, (outs DestRC:$dst), (ins SrcRC:$src),
    373          !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
    374          []>, EVEX;
    375   def rm : AVX5128I<opc, MRMSrcMem, (outs DestRC:$dst), (ins x86memop:$src),
    376         !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),[]>, EVEX;
    377 }
    378 let ExeDomain = SSEPackedSingle in {
    379   defm VBROADCASTSSZ  : avx512_fp_broadcast<0x18, "vbroadcastss", VR512,
    380                                        VR128X, f32mem>,
    381                                        EVEX_V512, EVEX_CD8<32, CD8VT1>;
    382 }
    383 
    384 let ExeDomain = SSEPackedDouble in {
    385   defm VBROADCASTSDZ  : avx512_fp_broadcast<0x19, "vbroadcastsd", VR512,
    386                                        VR128X, f64mem>,
    387                                        EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
    388 }
    389 
    390 def : Pat<(v16f32 (X86VBroadcast (loadf32 addr:$src))),
    391           (VBROADCASTSSZrm addr:$src)>;
    392 def : Pat<(v8f64 (X86VBroadcast (loadf64 addr:$src))),
    393           (VBROADCASTSDZrm addr:$src)>;
    394 
    395 def : Pat<(int_x86_avx512_vbroadcast_ss_512 addr:$src),
    396           (VBROADCASTSSZrm addr:$src)>;
    397 def : Pat<(int_x86_avx512_vbroadcast_sd_512 addr:$src),
    398           (VBROADCASTSDZrm addr:$src)>;
    399 
    400 multiclass avx512_int_broadcast_reg<bits<8> opc, string OpcodeStr,
    401                           RegisterClass SrcRC, RegisterClass KRC> {
    402   def Zrr : AVX5128I<opc, MRMSrcReg, (outs VR512:$dst), (ins SrcRC:$src),
    403                    !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
    404                    []>, EVEX, EVEX_V512;
    405   def Zkrr : AVX5128I<opc, MRMSrcReg, (outs VR512:$dst), 
    406                    (ins KRC:$mask, SrcRC:$src),
    407                    !strconcat(OpcodeStr, 
    408                         " \t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}"),
    409                    []>, EVEX, EVEX_V512, EVEX_KZ;
    410 }
    411 
    412 defm VPBROADCASTDr  : avx512_int_broadcast_reg<0x7C, "vpbroadcastd", GR32, VK16WM>;
    413 defm VPBROADCASTQr  : avx512_int_broadcast_reg<0x7C, "vpbroadcastq", GR64, VK8WM>,
    414                                             VEX_W;
    415                                             
    416 def : Pat <(v16i32 (X86vzext VK16WM:$mask)),
    417            (VPBROADCASTDrZkrr VK16WM:$mask, (i32 (MOV32ri 0x1)))>;
    418 
    419 def : Pat <(v8i64 (X86vzext VK8WM:$mask)),
    420            (VPBROADCASTQrZkrr VK8WM:$mask, (i64 (MOV64ri 0x1)))>;
    421 
    422 def : Pat<(v16i32 (X86VBroadcast (i32 GR32:$src))),
    423         (VPBROADCASTDrZrr GR32:$src)>;
    424 def : Pat<(v16i32 (X86VBroadcastm VK16WM:$mask, (i32 GR32:$src))),
    425         (VPBROADCASTDrZkrr VK16WM:$mask, GR32:$src)>;
    426 def : Pat<(v8i64 (X86VBroadcast (i64 GR64:$src))),
    427         (VPBROADCASTQrZrr GR64:$src)>;
    428 def : Pat<(v8i64 (X86VBroadcastm VK8WM:$mask, (i64 GR64:$src))),
    429         (VPBROADCASTQrZkrr VK8WM:$mask, GR64:$src)>;
    430 
    431 def : Pat<(v16i32 (int_x86_avx512_pbroadcastd_i32_512 (i32 GR32:$src))),
    432         (VPBROADCASTDrZrr GR32:$src)>;
    433 def : Pat<(v8i64 (int_x86_avx512_pbroadcastq_i64_512 (i64 GR64:$src))),
    434         (VPBROADCASTQrZrr GR64:$src)>;
    435 
    436 def : Pat<(v16i32 (int_x86_avx512_mask_pbroadcast_d_gpr_512 (i32 GR32:$src),
    437                    (v16i32 immAllZerosV), (i16 GR16:$mask))),
    438           (VPBROADCASTDrZkrr (COPY_TO_REGCLASS GR16:$mask, VK16WM), GR32:$src)>;
    439 def : Pat<(v8i64 (int_x86_avx512_mask_pbroadcast_q_gpr_512 (i64 GR64:$src),
    440                    (bc_v8i64 (v16i32 immAllZerosV)), (i8 GR8:$mask))),
    441           (VPBROADCASTQrZkrr (COPY_TO_REGCLASS GR8:$mask, VK8WM), GR64:$src)>;
    442 
    443 multiclass avx512_int_broadcast_rm<bits<8> opc, string OpcodeStr,
    444                           X86MemOperand x86memop, PatFrag ld_frag,
    445                           RegisterClass DstRC, ValueType OpVT, ValueType SrcVT,
    446                           RegisterClass KRC> {
    447   def rr : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst), (ins VR128X:$src),
    448                   !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
    449                   [(set DstRC:$dst,
    450                     (OpVT (X86VBroadcast (SrcVT VR128X:$src))))]>, EVEX;
    451   def krr : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst), (ins KRC:$mask,
    452                                                          VR128X:$src),
    453                     !strconcat(OpcodeStr, 
    454                     " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"),
    455                     [(set DstRC:$dst,
    456                       (OpVT (X86VBroadcastm KRC:$mask, (SrcVT VR128X:$src))))]>,
    457                     EVEX, EVEX_KZ;
    458   let mayLoad = 1 in {
    459   def rm : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src),
    460                   !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
    461                   [(set DstRC:$dst, 
    462                     (OpVT (X86VBroadcast (ld_frag addr:$src))))]>, EVEX;
    463   def krm : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst), (ins KRC:$mask,
    464                                                          x86memop:$src),
    465                   !strconcat(OpcodeStr, 
    466                       " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"),
    467                   [(set DstRC:$dst, (OpVT (X86VBroadcastm KRC:$mask, 
    468                                      (ld_frag addr:$src))))]>, EVEX, EVEX_KZ;
    469   }
    470 }
    471 
    472 defm VPBROADCASTDZ  : avx512_int_broadcast_rm<0x58, "vpbroadcastd", i32mem,
    473                       loadi32, VR512, v16i32, v4i32, VK16WM>,
    474                       EVEX_V512, EVEX_CD8<32, CD8VT1>;
    475 defm VPBROADCASTQZ  : avx512_int_broadcast_rm<0x59, "vpbroadcastq", i64mem,
    476                       loadi64, VR512, v8i64, v2i64, VK8WM>,  EVEX_V512, VEX_W,
    477                       EVEX_CD8<64, CD8VT1>;
    478 
    479 multiclass avx512_int_subvec_broadcast_rm<bits<8> opc, string OpcodeStr,
    480                           X86MemOperand x86memop, PatFrag ld_frag,
    481                           RegisterClass KRC> {
    482   let mayLoad = 1 in {
    483   def rm : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst), (ins x86memop:$src),
    484                   !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
    485                   []>, EVEX;
    486   def krm : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst), (ins KRC:$mask,
    487                                                          x86memop:$src),
    488                   !strconcat(OpcodeStr,
    489                       " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"),
    490                   []>, EVEX, EVEX_KZ;
    491   }
    492 }
    493 
    494 defm VBROADCASTI32X4 : avx512_int_subvec_broadcast_rm<0x5a, "vbroadcasti32x4",
    495                        i128mem, loadv2i64, VK16WM>,
    496                        EVEX_V512, EVEX_CD8<32, CD8VT4>;
    497 defm VBROADCASTI64X4 : avx512_int_subvec_broadcast_rm<0x5b, "vbroadcasti64x4",
    498                        i256mem, loadv4i64, VK16WM>, VEX_W,
    499                        EVEX_V512, EVEX_CD8<64, CD8VT4>;
    500 
    501 def : Pat<(v16i32 (int_x86_avx512_pbroadcastd_512 (v4i32 VR128X:$src))),
    502           (VPBROADCASTDZrr VR128X:$src)>;
    503 def : Pat<(v8i64 (int_x86_avx512_pbroadcastq_512 (v2i64 VR128X:$src))),
    504           (VPBROADCASTQZrr VR128X:$src)>;
    505 
    506 def : Pat<(v16f32 (X86VBroadcast (v4f32 VR128X:$src))),
    507           (VBROADCASTSSZrr VR128X:$src)>;
    508 def : Pat<(v8f64 (X86VBroadcast (v2f64 VR128X:$src))),
    509           (VBROADCASTSDZrr VR128X:$src)>;
    510 
    511 def : Pat<(v16f32 (int_x86_avx512_vbroadcast_ss_ps_512 (v4f32 VR128X:$src))),
    512           (VBROADCASTSSZrr VR128X:$src)>;
    513 def : Pat<(v8f64 (int_x86_avx512_vbroadcast_sd_pd_512 (v2f64 VR128X:$src))),
    514           (VBROADCASTSDZrr VR128X:$src)>;
    515     
    516 // Provide fallback in case the load node that is used in the patterns above
    517 // is used by additional users, which prevents the pattern selection.
    518 def : Pat<(v16f32 (X86VBroadcast FR32X:$src)),
    519           (VBROADCASTSSZrr (COPY_TO_REGCLASS FR32X:$src, VR128X))>;
    520 def : Pat<(v8f64 (X86VBroadcast FR64X:$src)),
    521           (VBROADCASTSDZrr (COPY_TO_REGCLASS FR64X:$src, VR128X))>;
    522 
    523 
    524 let Predicates = [HasAVX512] in {
    525 def : Pat<(v8i32 (X86VBroadcastm (v8i1 VK8WM:$mask), (loadi32 addr:$src))),
    526            (EXTRACT_SUBREG 
    527               (v16i32 (VPBROADCASTDZkrm (COPY_TO_REGCLASS VK8WM:$mask, VK16WM),
    528                        addr:$src)), sub_ymm)>;
    529 }
    530 //===----------------------------------------------------------------------===//
    531 // AVX-512 BROADCAST MASK TO VECTOR REGISTER
    532 //---
    533 
    534 multiclass avx512_mask_broadcast<bits<8> opc, string OpcodeStr,
    535                        RegisterClass DstRC, RegisterClass KRC,
    536                        ValueType OpVT, ValueType SrcVT> {
    537 def rr : AVX512XS8I<opc, MRMDestReg, (outs DstRC:$dst), (ins KRC:$src),
    538                   !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
    539                   []>, EVEX;
    540 }
    541 
    542 let Predicates = [HasCDI] in {
    543 defm VPBROADCASTMW2D : avx512_mask_broadcast<0x3A, "vpbroadcastmw2d", VR512,
    544                                              VK16, v16i32, v16i1>, EVEX_V512;
    545 defm VPBROADCASTMB2Q : avx512_mask_broadcast<0x2A, "vpbroadcastmb2q", VR512,
    546                                             VK8, v8i64, v8i1>, EVEX_V512, VEX_W;
    547 }
    548 
    549 //===----------------------------------------------------------------------===//
    550 // AVX-512 - VPERM
    551 //
    552 // -- immediate form --
    553 multiclass avx512_perm_imm<bits<8> opc, string OpcodeStr, RegisterClass RC,
    554                          SDNode OpNode, PatFrag mem_frag, 
    555                          X86MemOperand x86memop, ValueType OpVT> {
    556   def ri : AVX512AIi8<opc, MRMSrcReg, (outs RC:$dst),
    557                      (ins RC:$src1, i8imm:$src2),
    558                      !strconcat(OpcodeStr,
    559                          " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
    560                      [(set RC:$dst,
    561                        (OpVT (OpNode RC:$src1, (i8 imm:$src2))))]>,
    562                      EVEX;
    563   def mi : AVX512AIi8<opc, MRMSrcMem, (outs RC:$dst),
    564                      (ins x86memop:$src1, i8imm:$src2),
    565                      !strconcat(OpcodeStr,
    566                          " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
    567                      [(set RC:$dst,
    568                        (OpVT (OpNode (mem_frag addr:$src1),
    569                               (i8 imm:$src2))))]>, EVEX;
    570 }
    571 
    572 defm VPERMQZ  : avx512_perm_imm<0x00, "vpermq", VR512, X86VPermi, memopv8i64,
    573                         i512mem, v8i64>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
    574 let ExeDomain = SSEPackedDouble in 
    575 defm VPERMPDZ  : avx512_perm_imm<0x01, "vpermpd", VR512, X86VPermi, memopv8f64, 
    576                         f512mem, v8f64>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
    577 
    578 // -- VPERM - register form --
    579 multiclass avx512_perm<bits<8> opc, string OpcodeStr, RegisterClass RC, 
    580                      PatFrag mem_frag, X86MemOperand x86memop, ValueType OpVT> {
    581 
    582   def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst),
    583                    (ins RC:$src1, RC:$src2),
    584                    !strconcat(OpcodeStr,
    585                        " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
    586                    [(set RC:$dst,
    587                      (OpVT (X86VPermv RC:$src1, RC:$src2)))]>, EVEX_4V;
    588 
    589   def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
    590                    (ins RC:$src1, x86memop:$src2),
    591                    !strconcat(OpcodeStr,
    592                        " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
    593                    [(set RC:$dst,
    594                      (OpVT (X86VPermv RC:$src1, (mem_frag addr:$src2))))]>,
    595                      EVEX_4V;
    596 }
    597 
    598 defm VPERMDZ   : avx512_perm<0x36, "vpermd",  VR512,  memopv16i32, i512mem,
    599                            v16i32>, EVEX_V512, EVEX_CD8<32, CD8VF>;
    600 defm VPERMQZ   : avx512_perm<0x36, "vpermq",  VR512,  memopv8i64,  i512mem, 
    601                            v8i64>,  EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
    602 let ExeDomain = SSEPackedSingle in
    603 defm VPERMPSZ  : avx512_perm<0x16, "vpermps", VR512,  memopv16f32, f512mem,
    604                            v16f32>, EVEX_V512, EVEX_CD8<32, CD8VF>;
    605 let ExeDomain = SSEPackedDouble in
    606 defm VPERMPDZ  : avx512_perm<0x16, "vpermpd", VR512,  memopv8f64, f512mem, 
    607                            v8f64>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
    608 
    609 // -- VPERM2I - 3 source operands form --
    610 multiclass avx512_perm_3src<bits<8> opc, string OpcodeStr, RegisterClass RC,
    611                           PatFrag mem_frag, X86MemOperand x86memop,
    612                           SDNode OpNode, ValueType OpVT, RegisterClass KRC> {
    613 let Constraints = "$src1 = $dst" in {
    614   def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst),
    615                    (ins RC:$src1, RC:$src2, RC:$src3),
    616                    !strconcat(OpcodeStr,
    617                        " \t{$src3, $src2, $dst|$dst, $src2, $src3}"),
    618                    [(set RC:$dst,
    619                      (OpVT (OpNode RC:$src1, RC:$src2, RC:$src3)))]>,
    620                     EVEX_4V;
    621 
    622   def rrk : AVX5128I<opc, MRMSrcReg, (outs RC:$dst),
    623                    (ins RC:$src1, KRC:$mask, RC:$src2, RC:$src3),
    624                    !strconcat(OpcodeStr,
    625                        " \t{$src3, $src2, $dst {${mask}}|"
    626                        "$dst {${mask}}, $src2, $src3}"),
    627                    [(set RC:$dst, (OpVT (vselect KRC:$mask,
    628                                            (OpNode RC:$src1, RC:$src2,
    629                                               RC:$src3),
    630                                            RC:$src1)))]>,
    631                     EVEX_4V, EVEX_K;
    632 
    633   let AddedComplexity = 30 in // Prefer over VMOV*rrkz Pat<>
    634     def rrkz : AVX5128I<opc, MRMSrcReg, (outs RC:$dst),
    635                    (ins RC:$src1, KRC:$mask, RC:$src2, RC:$src3),
    636                    !strconcat(OpcodeStr,
    637                        " \t{$src3, $src2, $dst {${mask}} {z} |",
    638                        "$dst {${mask}} {z}, $src2, $src3}"),
    639                    [(set RC:$dst, (OpVT (vselect KRC:$mask,
    640                                            (OpNode RC:$src1, RC:$src2,
    641                                               RC:$src3),
    642                                            (OpVT (bitconvert
    643                                               (v16i32 immAllZerosV))))))]>,
    644                     EVEX_4V, EVEX_KZ;
    645 
    646   def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
    647                    (ins RC:$src1, RC:$src2, x86memop:$src3),
    648                    !strconcat(OpcodeStr,
    649                     " \t{$src3, $src2, $dst|$dst, $src2, $src3}"),
    650                    [(set RC:$dst,
    651                      (OpVT (OpNode RC:$src1, RC:$src2,
    652                       (mem_frag addr:$src3))))]>, EVEX_4V;
    653 
    654   def rmk : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
    655                    (ins RC:$src1, KRC:$mask, RC:$src2, x86memop:$src3),
    656                    !strconcat(OpcodeStr,
    657                     " \t{$src3, $src2, $dst {${mask}}|"
    658                     "$dst {${mask}}, $src2, $src3}"),
    659                    [(set RC:$dst,
    660                        (OpVT (vselect KRC:$mask,
    661                                       (OpNode RC:$src1, RC:$src2,
    662                                          (mem_frag addr:$src3)),
    663                                       RC:$src1)))]>,
    664                     EVEX_4V, EVEX_K;
    665 
    666   let AddedComplexity = 10 in // Prefer over the rrkz variant
    667     def rmkz : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
    668                    (ins RC:$src1, KRC:$mask, RC:$src2, x86memop:$src3),
    669                    !strconcat(OpcodeStr,
    670                     " \t{$src3, $src2, $dst {${mask}} {z}|"
    671                     "$dst {${mask}} {z}, $src2, $src3}"),
    672                    [(set RC:$dst,
    673                      (OpVT (vselect KRC:$mask,
    674                                     (OpNode RC:$src1, RC:$src2,
    675                                             (mem_frag addr:$src3)),
    676                                     (OpVT (bitconvert
    677                                        (v16i32 immAllZerosV))))))]>,
    678                     EVEX_4V, EVEX_KZ;
    679   }
    680 }
    681 defm VPERMI2D  : avx512_perm_3src<0x76, "vpermi2d",  VR512, memopv16i32,
    682                                   i512mem, X86VPermiv3, v16i32, VK16WM>,
    683                  EVEX_V512, EVEX_CD8<32, CD8VF>;
    684 defm VPERMI2Q  : avx512_perm_3src<0x76, "vpermi2q",  VR512, memopv8i64,
    685                                   i512mem, X86VPermiv3, v8i64, VK8WM>,
    686                  EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
    687 defm VPERMI2PS : avx512_perm_3src<0x77, "vpermi2ps",  VR512, memopv16f32,
    688                                   i512mem, X86VPermiv3, v16f32, VK16WM>,
    689                  EVEX_V512, EVEX_CD8<32, CD8VF>;
    690 defm VPERMI2PD : avx512_perm_3src<0x77, "vpermi2pd",  VR512, memopv8f64,
    691                                   i512mem, X86VPermiv3, v8f64, VK8WM>,
    692                   EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
    693 
    694 multiclass avx512_perm_table_3src<bits<8> opc, string Suffix, RegisterClass RC,
    695                           PatFrag mem_frag, X86MemOperand x86memop,
    696                           SDNode OpNode, ValueType OpVT, RegisterClass KRC,
    697                           ValueType MaskVT, RegisterClass MRC> :
    698         avx512_perm_3src<opc, "vpermt2"##Suffix, RC, mem_frag, x86memop, OpNode,
    699                          OpVT, KRC> {
    700   def : Pat<(OpVT (!cast<Intrinsic>("int_x86_avx512_mask_vpermt_"##Suffix##"_512")
    701                      VR512:$idx, VR512:$src1, VR512:$src2, -1)),
    702             (!cast<Instruction>(NAME#rr) VR512:$src1, VR512:$idx, VR512:$src2)>;
    703 
    704   def : Pat<(OpVT (!cast<Intrinsic>("int_x86_avx512_mask_vpermt_"##Suffix##"_512")
    705                      VR512:$idx, VR512:$src1, VR512:$src2, MRC:$mask)),
    706             (!cast<Instruction>(NAME#rrk) VR512:$src1,
    707               (MaskVT (COPY_TO_REGCLASS MRC:$mask, KRC)), VR512:$idx, VR512:$src2)>;
    708 }
    709 
    710 defm VPERMT2D  : avx512_perm_table_3src<0x7E, "d",  VR512, memopv16i32, i512mem,
    711                                X86VPermv3, v16i32, VK16WM, v16i1, GR16>,
    712                  EVEX_V512, EVEX_CD8<32, CD8VF>;
    713 defm VPERMT2Q  : avx512_perm_table_3src<0x7E, "q",  VR512, memopv8i64, i512mem,
    714                                X86VPermv3, v8i64, VK8WM, v8i1, GR8>,
    715                  EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
    716 defm VPERMT2PS : avx512_perm_table_3src<0x7F, "ps",  VR512, memopv16f32, i512mem,
    717                                X86VPermv3, v16f32, VK16WM, v16i1, GR16>,
    718                  EVEX_V512, EVEX_CD8<32, CD8VF>;
    719 defm VPERMT2PD : avx512_perm_table_3src<0x7F, "pd",  VR512, memopv8f64, i512mem,
    720                                X86VPermv3, v8f64, VK8WM, v8i1, GR8>,
    721                  EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
    722 
    723 //===----------------------------------------------------------------------===//
    724 // AVX-512 - BLEND using mask
    725 //
    726 multiclass avx512_blendmask<bits<8> opc, string OpcodeStr,
    727                           RegisterClass KRC, RegisterClass RC,
    728                           X86MemOperand x86memop, PatFrag mem_frag,
    729                           SDNode OpNode, ValueType vt> {
    730   def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst),
    731              (ins KRC:$mask, RC:$src1, RC:$src2),
    732              !strconcat(OpcodeStr,
    733              " \t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"),
    734              [(set RC:$dst, (OpNode KRC:$mask, (vt RC:$src2),
    735                  (vt RC:$src1)))]>, EVEX_4V, EVEX_K;
    736   let mayLoad = 1 in
    737   def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
    738              (ins KRC:$mask, RC:$src1, x86memop:$src2),
    739              !strconcat(OpcodeStr,
    740              " \t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"),
    741              []>, EVEX_4V, EVEX_K;
    742 }
    743 
    744 let ExeDomain = SSEPackedSingle in
    745 defm VBLENDMPSZ : avx512_blendmask<0x65, "vblendmps", 
    746                               VK16WM, VR512, f512mem,
    747                               memopv16f32, vselect, v16f32>, 
    748                               EVEX_CD8<32, CD8VF>, EVEX_V512;
    749 let ExeDomain = SSEPackedDouble in
    750 defm VBLENDMPDZ : avx512_blendmask<0x65, "vblendmpd", 
    751                               VK8WM, VR512, f512mem,
    752                               memopv8f64, vselect, v8f64>, 
    753                               VEX_W, EVEX_CD8<64, CD8VF>, EVEX_V512;
    754 
    755 def : Pat<(v16f32 (int_x86_avx512_mask_blend_ps_512 (v16f32 VR512:$src1),
    756                  (v16f32 VR512:$src2), (i16 GR16:$mask))),
    757         (VBLENDMPSZrr (COPY_TO_REGCLASS GR16:$mask, VK16WM),
    758          VR512:$src1, VR512:$src2)>;
    759 
    760 def : Pat<(v8f64 (int_x86_avx512_mask_blend_pd_512 (v8f64 VR512:$src1),
    761                  (v8f64 VR512:$src2), (i8 GR8:$mask))),
    762         (VBLENDMPDZrr (COPY_TO_REGCLASS GR8:$mask, VK8WM),
    763          VR512:$src1, VR512:$src2)>;
    764 
    765 defm VPBLENDMDZ : avx512_blendmask<0x64, "vpblendmd", 
    766                               VK16WM, VR512, f512mem, 
    767                               memopv16i32, vselect, v16i32>, 
    768                               EVEX_CD8<32, CD8VF>, EVEX_V512;
    769 
    770 defm VPBLENDMQZ : avx512_blendmask<0x64, "vpblendmq", 
    771                               VK8WM, VR512, f512mem, 
    772                               memopv8i64, vselect, v8i64>, 
    773                               VEX_W, EVEX_CD8<64, CD8VF>, EVEX_V512;
    774 
    775 def : Pat<(v16i32 (int_x86_avx512_mask_blend_d_512 (v16i32 VR512:$src1),
    776                  (v16i32 VR512:$src2), (i16 GR16:$mask))),
    777         (VPBLENDMDZrr (COPY_TO_REGCLASS GR16:$mask, VK16),
    778          VR512:$src1, VR512:$src2)>;
    779 
    780 def : Pat<(v8i64 (int_x86_avx512_mask_blend_q_512 (v8i64 VR512:$src1),
    781                  (v8i64 VR512:$src2), (i8 GR8:$mask))),
    782         (VPBLENDMQZrr (COPY_TO_REGCLASS GR8:$mask, VK8),
    783          VR512:$src1, VR512:$src2)>;
    784 
    785 let Predicates = [HasAVX512] in {
    786 def : Pat<(v8f32 (vselect (v8i1 VK8WM:$mask), (v8f32 VR256X:$src1),
    787                             (v8f32 VR256X:$src2))),
    788             (EXTRACT_SUBREG 
    789               (v16f32 (VBLENDMPSZrr (COPY_TO_REGCLASS VK8WM:$mask, VK16WM), 
    790             (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)),
    791             (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>;
    792 
    793 def : Pat<(v8i32 (vselect (v8i1 VK8WM:$mask), (v8i32 VR256X:$src1),
    794                             (v8i32 VR256X:$src2))),
    795             (EXTRACT_SUBREG 
    796                 (v16i32 (VPBLENDMDZrr (COPY_TO_REGCLASS VK8WM:$mask, VK16WM), 
    797             (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)),
    798             (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>;
    799 }
    800 //===----------------------------------------------------------------------===//
    801 // Compare Instructions
    802 //===----------------------------------------------------------------------===//
    803 
    804 // avx512_cmp_scalar - AVX512 CMPSS and CMPSD
    805 multiclass avx512_cmp_scalar<RegisterClass RC, X86MemOperand x86memop,
    806                             Operand CC, SDNode OpNode, ValueType VT,
    807                             PatFrag ld_frag, string asm, string asm_alt> {
    808   def rr : AVX512Ii8<0xC2, MRMSrcReg,
    809                 (outs VK1:$dst), (ins RC:$src1, RC:$src2, CC:$cc), asm,
    810                 [(set VK1:$dst, (OpNode (VT RC:$src1), RC:$src2, imm:$cc))],
    811                 IIC_SSE_ALU_F32S_RR>, EVEX_4V;
    812   def rm : AVX512Ii8<0xC2, MRMSrcMem,
    813                 (outs VK1:$dst), (ins RC:$src1, x86memop:$src2, CC:$cc), asm,
    814                 [(set VK1:$dst, (OpNode (VT RC:$src1),
    815                 (ld_frag addr:$src2), imm:$cc))], IIC_SSE_ALU_F32P_RM>, EVEX_4V;
    816   let isAsmParserOnly = 1, hasSideEffects = 0 in {
    817     def rri_alt : AVX512Ii8<0xC2, MRMSrcReg,
    818                (outs VK1:$dst), (ins RC:$src1, RC:$src2, i8imm:$cc),
    819                asm_alt, [], IIC_SSE_ALU_F32S_RR>, EVEX_4V;
    820     def rmi_alt : AVX512Ii8<0xC2, MRMSrcMem,
    821                (outs VK1:$dst), (ins RC:$src1, x86memop:$src2, i8imm:$cc),
    822                asm_alt, [], IIC_SSE_ALU_F32P_RM>, EVEX_4V;
    823   }
    824 }
    825 
    826 let Predicates = [HasAVX512] in {
    827 defm VCMPSSZ : avx512_cmp_scalar<FR32X, f32mem, AVXCC, X86cmpms, f32, loadf32,
    828                  "vcmp${cc}ss\t{$src2, $src1, $dst|$dst, $src1, $src2}",
    829                  "vcmpss\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}">,
    830                  XS;
    831 defm VCMPSDZ : avx512_cmp_scalar<FR64X, f64mem, AVXCC, X86cmpms, f64, loadf64,
    832                  "vcmp${cc}sd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
    833                  "vcmpsd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}">,
    834                  XD, VEX_W;
    835 }
    836 
    837 multiclass avx512_icmp_packed<bits<8> opc, string OpcodeStr, RegisterClass KRC, 
    838               RegisterClass RC, X86MemOperand x86memop, PatFrag memop_frag, 
    839               SDNode OpNode, ValueType vt> {
    840   def rr : AVX512BI<opc, MRMSrcReg,
    841              (outs KRC:$dst), (ins RC:$src1, RC:$src2), 
    842              !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
    843              [(set KRC:$dst, (OpNode (vt RC:$src1), (vt RC:$src2)))], 
    844              IIC_SSE_ALU_F32P_RR>, EVEX_4V;
    845   def rm : AVX512BI<opc, MRMSrcMem,
    846              (outs KRC:$dst), (ins RC:$src1, x86memop:$src2), 
    847              !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
    848              [(set KRC:$dst, (OpNode (vt RC:$src1), (memop_frag addr:$src2)))],
    849              IIC_SSE_ALU_F32P_RM>, EVEX_4V;
    850 }
    851 
    852 defm VPCMPEQDZ : avx512_icmp_packed<0x76, "vpcmpeqd", VK16, VR512, i512mem, 
    853                            memopv16i32, X86pcmpeqm, v16i32>, EVEX_V512,
    854                            EVEX_CD8<32, CD8VF>;
    855 defm VPCMPEQQZ : avx512_icmp_packed<0x29, "vpcmpeqq", VK8, VR512, i512mem, 
    856                            memopv8i64, X86pcmpeqm, v8i64>, T8PD, EVEX_V512,
    857                            VEX_W, EVEX_CD8<64, CD8VF>;
    858 
    859 defm VPCMPGTDZ : avx512_icmp_packed<0x66, "vpcmpgtd", VK16, VR512, i512mem, 
    860                            memopv16i32, X86pcmpgtm, v16i32>, EVEX_V512,
    861                            EVEX_CD8<32, CD8VF>;
    862 defm VPCMPGTQZ : avx512_icmp_packed<0x37, "vpcmpgtq", VK8, VR512, i512mem, 
    863                            memopv8i64, X86pcmpgtm, v8i64>, T8PD, EVEX_V512,
    864                            VEX_W, EVEX_CD8<64, CD8VF>;
    865 
    866 def : Pat<(v8i1 (X86pcmpgtm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))),
    867             (COPY_TO_REGCLASS (VPCMPGTDZrr 
    868             (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)),
    869             (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm))), VK8)>;
    870 
    871 def : Pat<(v8i1 (X86pcmpeqm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))),
    872             (COPY_TO_REGCLASS (VPCMPEQDZrr 
    873             (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)),
    874             (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm))), VK8)>;
    875 
    876 multiclass avx512_icmp_cc<bits<8> opc, RegisterClass WMRC, RegisterClass KRC,
    877               RegisterClass RC, X86MemOperand x86memop, PatFrag memop_frag, 
    878               SDNode OpNode, ValueType vt, Operand CC, string Suffix> {
    879   def rri : AVX512AIi8<opc, MRMSrcReg,
    880              (outs KRC:$dst), (ins RC:$src1, RC:$src2, CC:$cc),
    881              !strconcat("vpcmp${cc}", Suffix,
    882                         "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
    883              [(set KRC:$dst, (OpNode (vt RC:$src1), (vt RC:$src2), imm:$cc))], 
    884              IIC_SSE_ALU_F32P_RR>, EVEX_4V;
    885   def rmi : AVX512AIi8<opc, MRMSrcMem,
    886              (outs KRC:$dst), (ins RC:$src1, x86memop:$src2, CC:$cc),
    887              !strconcat("vpcmp${cc}", Suffix,
    888                         "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
    889              [(set KRC:$dst, (OpNode (vt RC:$src1), (memop_frag addr:$src2),
    890                               imm:$cc))], IIC_SSE_ALU_F32P_RM>, EVEX_4V;
    891   // Accept explicit immediate argument form instead of comparison code.
    892   let isAsmParserOnly = 1, hasSideEffects = 0 in {
    893     def rri_alt : AVX512AIi8<opc, MRMSrcReg,
    894                (outs KRC:$dst), (ins RC:$src1, RC:$src2, i8imm:$cc),
    895                !strconcat("vpcmp", Suffix,
    896                   "\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"),
    897                [], IIC_SSE_ALU_F32P_RR>, EVEX_4V;
    898     def rrik_alt : AVX512AIi8<opc, MRMSrcReg,
    899                (outs KRC:$dst), (ins WMRC:$mask, RC:$src1, RC:$src2, i8imm:$cc),
    900                !strconcat("vpcmp", Suffix,
    901                   "\t{$cc, $src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2, $cc}"),
    902                [], IIC_SSE_ALU_F32P_RR>, EVEX_4V, EVEX_K;
    903     def rmi_alt : AVX512AIi8<opc, MRMSrcMem,
    904                (outs KRC:$dst), (ins RC:$src1, x86memop:$src2, i8imm:$cc),
    905                !strconcat("vpcmp", Suffix,
    906                   "\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"),
    907                [], IIC_SSE_ALU_F32P_RM>, EVEX_4V;
    908     def rmik_alt : AVX512AIi8<opc, MRMSrcMem,
    909                (outs KRC:$dst), (ins WMRC:$mask, RC:$src1, x86memop:$src2, i8imm:$cc),
    910                !strconcat("vpcmp", Suffix,
    911                   "\t{$cc, $src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2, $cc}"),
    912                [], IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K;
    913   }
    914 }
    915 
    916 defm VPCMPDZ :  avx512_icmp_cc<0x1F, VK16WM, VK16, VR512, i512mem, memopv16i32,
    917                                X86cmpm, v16i32, AVXCC, "d">,
    918                 EVEX_V512, EVEX_CD8<32, CD8VF>;
    919 defm VPCMPUDZ : avx512_icmp_cc<0x1E, VK16WM, VK16, VR512, i512mem, memopv16i32,
    920                                X86cmpmu, v16i32, AVXCC, "ud">,
    921                 EVEX_V512, EVEX_CD8<32, CD8VF>;
    922 
    923 defm VPCMPQZ :  avx512_icmp_cc<0x1F, VK8WM, VK8, VR512, i512mem, memopv8i64,
    924                                X86cmpm, v8i64, AVXCC, "q">,
    925                 VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
    926 defm VPCMPUQZ : avx512_icmp_cc<0x1E, VK8WM, VK8, VR512, i512mem, memopv8i64,
    927                                X86cmpmu, v8i64, AVXCC, "uq">,
    928                 VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
    929 
    930 // avx512_cmp_packed - compare packed instructions
    931 multiclass avx512_cmp_packed<RegisterClass KRC, RegisterClass RC,
    932                            X86MemOperand x86memop, ValueType vt,
    933                            string suffix, Domain d> {
    934   def rri : AVX512PIi8<0xC2, MRMSrcReg,
    935              (outs KRC:$dst), (ins RC:$src1, RC:$src2, AVXCC:$cc),
    936              !strconcat("vcmp${cc}", suffix,
    937                         " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
    938              [(set KRC:$dst, (X86cmpm (vt RC:$src1), (vt RC:$src2), imm:$cc))], d>;
    939   def rrib: AVX512PIi8<0xC2, MRMSrcReg,
    940              (outs KRC:$dst), (ins RC:$src1, RC:$src2, AVXCC:$cc),
    941      !strconcat("vcmp${cc}", suffix,
    942                 " \t{{sae}, $src2, $src1, $dst|$dst, $src1, $src2, {sae}}"),
    943                 [], d>, EVEX_B;
    944   def rmi : AVX512PIi8<0xC2, MRMSrcMem,
    945              (outs KRC:$dst), (ins RC:$src1, x86memop:$src2, AVXCC:$cc),
    946               !strconcat("vcmp${cc}", suffix,
    947                          " \t{$src2, $src1, $dst|$dst, $src1, $src2, $cc}"),
    948              [(set KRC:$dst,
    949               (X86cmpm (vt RC:$src1), (memop addr:$src2), imm:$cc))], d>;
    950 
    951   // Accept explicit immediate argument form instead of comparison code.
    952   let isAsmParserOnly = 1, hasSideEffects = 0 in {
    953     def rri_alt : AVX512PIi8<0xC2, MRMSrcReg,
    954                (outs KRC:$dst), (ins RC:$src1, RC:$src2, i8imm:$cc),
    955               !strconcat("vcmp", suffix,
    956                         " \t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"), [], d>;
    957     def rmi_alt : AVX512PIi8<0xC2, MRMSrcMem,
    958                (outs KRC:$dst), (ins RC:$src1, x86memop:$src2, i8imm:$cc),
    959               !strconcat("vcmp", suffix,
    960                         " \t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"), [], d>;
    961   }
    962 }
    963 
    964 defm VCMPPSZ : avx512_cmp_packed<VK16, VR512, f512mem, v16f32,
    965                "ps", SSEPackedSingle>, PS, EVEX_4V, EVEX_V512,
    966                EVEX_CD8<32, CD8VF>;
    967 defm VCMPPDZ : avx512_cmp_packed<VK8, VR512, f512mem, v8f64,
    968                "pd", SSEPackedDouble>, PD, EVEX_4V, VEX_W, EVEX_V512,
    969                EVEX_CD8<64, CD8VF>;
    970 
    971 def : Pat<(v8i1 (X86cmpm (v8f32 VR256X:$src1), (v8f32 VR256X:$src2), imm:$cc)),
    972           (COPY_TO_REGCLASS (VCMPPSZrri
    973             (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)),
    974             (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)),
    975             imm:$cc), VK8)>;
    976 def : Pat<(v8i1 (X86cmpm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2), imm:$cc)),
    977           (COPY_TO_REGCLASS (VPCMPDZrri
    978             (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)),
    979             (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)),
    980             imm:$cc), VK8)>;
    981 def : Pat<(v8i1 (X86cmpmu (v8i32 VR256X:$src1), (v8i32 VR256X:$src2), imm:$cc)),
    982           (COPY_TO_REGCLASS (VPCMPUDZrri
    983             (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)),
    984             (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)),
    985             imm:$cc), VK8)>;
    986 
    987 def : Pat<(i16 (int_x86_avx512_mask_cmp_ps_512 (v16f32 VR512:$src1),
    988                 (v16f32 VR512:$src2), imm:$cc, (i16 -1),
    989                  FROUND_NO_EXC)),
    990           (COPY_TO_REGCLASS (VCMPPSZrrib VR512:$src1, VR512:$src2,
    991                              (I8Imm imm:$cc)), GR16)>;
    992            
    993 def : Pat<(i8 (int_x86_avx512_mask_cmp_pd_512 (v8f64 VR512:$src1),
    994                 (v8f64 VR512:$src2), imm:$cc, (i8 -1),
    995                  FROUND_NO_EXC)),
    996           (COPY_TO_REGCLASS (VCMPPDZrrib VR512:$src1, VR512:$src2,
    997                              (I8Imm imm:$cc)), GR8)>;
    998 
    999 def : Pat<(i16 (int_x86_avx512_mask_cmp_ps_512 (v16f32 VR512:$src1),
   1000                 (v16f32 VR512:$src2), imm:$cc, (i16 -1),
   1001                 FROUND_CURRENT)),
   1002           (COPY_TO_REGCLASS (VCMPPSZrri VR512:$src1, VR512:$src2,
   1003                              (I8Imm imm:$cc)), GR16)>;
   1004 
   1005 def : Pat<(i8 (int_x86_avx512_mask_cmp_pd_512 (v8f64 VR512:$src1),
   1006                 (v8f64 VR512:$src2), imm:$cc, (i8 -1),
   1007                  FROUND_CURRENT)),
   1008           (COPY_TO_REGCLASS (VCMPPDZrri VR512:$src1, VR512:$src2,
   1009                              (I8Imm imm:$cc)), GR8)>;
   1010 
   1011 // Mask register copy, including
   1012 // - copy between mask registers
   1013 // - load/store mask registers
   1014 // - copy from GPR to mask register and vice versa
   1015 //
   1016 multiclass avx512_mask_mov<bits<8> opc_kk, bits<8> opc_km, bits<8> opc_mk,
   1017                          string OpcodeStr, RegisterClass KRC,
   1018                          ValueType vt, X86MemOperand x86memop> {
   1019   let hasSideEffects = 0 in {
   1020     def kk : I<opc_kk, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src),
   1021                !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), []>;
   1022     let mayLoad = 1 in
   1023     def km : I<opc_km, MRMSrcMem, (outs KRC:$dst), (ins x86memop:$src),
   1024                !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
   1025                [(set KRC:$dst, (vt (load addr:$src)))]>;
   1026     let mayStore = 1 in
   1027     def mk : I<opc_mk, MRMDestMem, (outs), (ins x86memop:$dst, KRC:$src),
   1028                !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), []>;
   1029   }
   1030 }
   1031 
   1032 multiclass avx512_mask_mov_gpr<bits<8> opc_kr, bits<8> opc_rk,
   1033                              string OpcodeStr,
   1034                              RegisterClass KRC, RegisterClass GRC> {
   1035   let hasSideEffects = 0 in {
   1036     def kr : I<opc_kr, MRMSrcReg, (outs KRC:$dst), (ins GRC:$src),
   1037                !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), []>;
   1038     def rk : I<opc_rk, MRMSrcReg, (outs GRC:$dst), (ins KRC:$src),
   1039                !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"), []>;
   1040   }
   1041 }
   1042 
   1043 let Predicates = [HasAVX512] in {
   1044   defm KMOVW : avx512_mask_mov<0x90, 0x90, 0x91, "kmovw", VK16, v16i1, i16mem>,
   1045                VEX, PS;
   1046   defm KMOVW : avx512_mask_mov_gpr<0x92, 0x93, "kmovw", VK16, GR32>,
   1047                VEX, PS;
   1048 }
   1049 
   1050 let Predicates = [HasAVX512] in {
   1051   // GR16 from/to 16-bit mask
   1052   def : Pat<(v16i1 (bitconvert (i16 GR16:$src))),
   1053             (KMOVWkr (SUBREG_TO_REG (i32 0), GR16:$src, sub_16bit))>;
   1054   def : Pat<(i16 (bitconvert (v16i1 VK16:$src))),
   1055             (EXTRACT_SUBREG (KMOVWrk VK16:$src), sub_16bit)>;
   1056 
   1057   // Store kreg in memory
   1058   def : Pat<(store (v16i1 VK16:$src), addr:$dst),
   1059             (KMOVWmk addr:$dst, VK16:$src)>;
   1060 
   1061   def : Pat<(store VK8:$src, addr:$dst),
   1062             (KMOVWmk addr:$dst, (COPY_TO_REGCLASS VK8:$src, VK16))>;
   1063 
   1064   def : Pat<(i1 (load addr:$src)),
   1065             (COPY_TO_REGCLASS (KMOVWkm addr:$src), VK1)>;
   1066 
   1067   def : Pat<(v8i1 (load addr:$src)),
   1068             (COPY_TO_REGCLASS (KMOVWkm addr:$src), VK8)>;
   1069 
   1070   def : Pat<(i1 (trunc (i32 GR32:$src))),
   1071             (COPY_TO_REGCLASS (KMOVWkr (AND32ri $src, (i32 1))), VK1)>;
   1072 
   1073   def : Pat<(i1 (trunc (i8 GR8:$src))),
   1074        (COPY_TO_REGCLASS
   1075         (KMOVWkr (AND32ri (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit), (i32 1))),
   1076        VK1)>;
   1077   def : Pat<(i1 (trunc (i16 GR16:$src))),
   1078        (COPY_TO_REGCLASS
   1079         (KMOVWkr (AND32ri (SUBREG_TO_REG (i32 0), $src, sub_16bit), (i32 1))),
   1080        VK1)>;
   1081             
   1082   def : Pat<(i32 (zext VK1:$src)),
   1083             (AND32ri (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), (i32 1))>;
   1084   def : Pat<(i8 (zext VK1:$src)),
   1085             (EXTRACT_SUBREG
   1086              (AND32ri (KMOVWrk
   1087                        (COPY_TO_REGCLASS VK1:$src, VK16)), (i32 1)), sub_8bit)>;
   1088   def : Pat<(i64 (zext VK1:$src)),
   1089             (AND64ri8 (SUBREG_TO_REG (i64 0),
   1090              (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), sub_32bit), (i64 1))>;
   1091   def : Pat<(i16 (zext VK1:$src)),
   1092             (EXTRACT_SUBREG
   1093              (AND32ri (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), (i32 1)),
   1094               sub_16bit)>;
   1095   def : Pat<(v16i1 (scalar_to_vector VK1:$src)),
   1096             (COPY_TO_REGCLASS VK1:$src, VK16)>;
   1097   def : Pat<(v8i1 (scalar_to_vector VK1:$src)),
   1098             (COPY_TO_REGCLASS VK1:$src, VK8)>;
   1099 }
   1100 // With AVX-512 only, 8-bit mask is promoted to 16-bit mask.
   1101 let Predicates = [HasAVX512] in {
   1102   // GR from/to 8-bit mask without native support
   1103   def : Pat<(v8i1 (bitconvert (i8 GR8:$src))),
   1104             (COPY_TO_REGCLASS
   1105               (KMOVWkr (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit)),
   1106               VK8)>;
   1107   def : Pat<(i8 (bitconvert (v8i1 VK8:$src))),
   1108             (EXTRACT_SUBREG
   1109               (KMOVWrk (COPY_TO_REGCLASS VK8:$src, VK16)),
   1110               sub_8bit)>;
   1111 
   1112   def : Pat<(i1 (X86Vextract VK16:$src, (iPTR 0))),
   1113             (COPY_TO_REGCLASS VK16:$src, VK1)>;
   1114   def : Pat<(i1 (X86Vextract VK8:$src, (iPTR 0))),
   1115             (COPY_TO_REGCLASS VK8:$src, VK1)>;
   1116 
   1117 }
   1118 
   1119 // Mask unary operation
   1120 // - KNOT
   1121 multiclass avx512_mask_unop<bits<8> opc, string OpcodeStr,
   1122                          RegisterClass KRC, SDPatternOperator OpNode> {
   1123   let Predicates = [HasAVX512] in
   1124     def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src),
   1125                !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
   1126                [(set KRC:$dst, (OpNode KRC:$src))]>;
   1127 }
   1128 
   1129 multiclass avx512_mask_unop_w<bits<8> opc, string OpcodeStr,
   1130                                SDPatternOperator OpNode> {
   1131   defm W : avx512_mask_unop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode>,
   1132                           VEX, PS;
   1133 }
   1134 
   1135 defm KNOT : avx512_mask_unop_w<0x44, "knot", not>;
   1136 
   1137 multiclass avx512_mask_unop_int<string IntName, string InstName> {
   1138   let Predicates = [HasAVX512] in
   1139     def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_w")
   1140                 (i16 GR16:$src)),
   1141               (COPY_TO_REGCLASS (!cast<Instruction>(InstName##"Wrr")
   1142               (v16i1 (COPY_TO_REGCLASS GR16:$src, VK16))), GR16)>;
   1143 }
   1144 defm : avx512_mask_unop_int<"knot", "KNOT">;
   1145 
   1146 def : Pat<(xor VK16:$src1, (v16i1 immAllOnesV)), (KNOTWrr VK16:$src1)>;
   1147 def : Pat<(xor VK8:$src1,  (v8i1 immAllOnesV)),
   1148           (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$src1, VK16)), VK8)>;
   1149 
   1150 // With AVX-512, 8-bit mask is promoted to 16-bit mask.
   1151 def : Pat<(not VK8:$src),
   1152           (COPY_TO_REGCLASS
   1153             (KNOTWrr (COPY_TO_REGCLASS VK8:$src, VK16)), VK8)>;
   1154 
   1155 // Mask binary operation
   1156 // - KAND, KANDN, KOR, KXNOR, KXOR
   1157 multiclass avx512_mask_binop<bits<8> opc, string OpcodeStr,
   1158                            RegisterClass KRC, SDPatternOperator OpNode> {
   1159   let Predicates = [HasAVX512] in
   1160     def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2),
   1161                !strconcat(OpcodeStr,
   1162                           " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   1163                [(set KRC:$dst, (OpNode KRC:$src1, KRC:$src2))]>;
   1164 }
   1165 
   1166 multiclass avx512_mask_binop_w<bits<8> opc, string OpcodeStr,
   1167                              SDPatternOperator OpNode> {
   1168   defm W : avx512_mask_binop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode>,
   1169                            VEX_4V, VEX_L, PS;
   1170 }
   1171 
   1172 def andn : PatFrag<(ops node:$i0, node:$i1), (and (not node:$i0), node:$i1)>;
   1173 def xnor : PatFrag<(ops node:$i0, node:$i1), (not (xor node:$i0, node:$i1))>;
   1174 
   1175 let isCommutable = 1 in {
   1176   defm KAND  : avx512_mask_binop_w<0x41, "kand",  and>;
   1177   let isCommutable = 0 in
   1178   defm KANDN : avx512_mask_binop_w<0x42, "kandn", andn>;
   1179   defm KOR   : avx512_mask_binop_w<0x45, "kor",   or>;
   1180   defm KXNOR : avx512_mask_binop_w<0x46, "kxnor", xnor>;
   1181   defm KXOR  : avx512_mask_binop_w<0x47, "kxor",  xor>;
   1182 }
   1183 
   1184 def : Pat<(xor VK1:$src1, VK1:$src2),
   1185      (COPY_TO_REGCLASS (KXORWrr (COPY_TO_REGCLASS VK1:$src1, VK16),
   1186                                 (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>;
   1187 
   1188 def : Pat<(or VK1:$src1, VK1:$src2),
   1189      (COPY_TO_REGCLASS (KORWrr (COPY_TO_REGCLASS VK1:$src1, VK16),
   1190                                (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>;
   1191 
   1192 def : Pat<(and VK1:$src1, VK1:$src2),
   1193      (COPY_TO_REGCLASS (KANDWrr (COPY_TO_REGCLASS VK1:$src1, VK16),
   1194                                 (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>;
   1195 
   1196 multiclass avx512_mask_binop_int<string IntName, string InstName> {
   1197   let Predicates = [HasAVX512] in
   1198     def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_w")
   1199                 (i16 GR16:$src1), (i16 GR16:$src2)),
   1200               (COPY_TO_REGCLASS (!cast<Instruction>(InstName##"Wrr")
   1201               (v16i1 (COPY_TO_REGCLASS GR16:$src1, VK16)),
   1202               (v16i1 (COPY_TO_REGCLASS GR16:$src2, VK16))), GR16)>;
   1203 }
   1204 
   1205 defm : avx512_mask_binop_int<"kand",  "KAND">;
   1206 defm : avx512_mask_binop_int<"kandn", "KANDN">;
   1207 defm : avx512_mask_binop_int<"kor",   "KOR">;
   1208 defm : avx512_mask_binop_int<"kxnor", "KXNOR">;
   1209 defm : avx512_mask_binop_int<"kxor",  "KXOR">;
   1210 
   1211 // With AVX-512, 8-bit mask is promoted to 16-bit mask.
   1212 multiclass avx512_binop_pat<SDPatternOperator OpNode, Instruction Inst> {
   1213   let Predicates = [HasAVX512] in
   1214     def : Pat<(OpNode VK8:$src1, VK8:$src2),
   1215               (COPY_TO_REGCLASS
   1216                 (Inst (COPY_TO_REGCLASS VK8:$src1, VK16),
   1217                       (COPY_TO_REGCLASS VK8:$src2, VK16)), VK8)>;
   1218 }
   1219 
   1220 defm : avx512_binop_pat<and,  KANDWrr>;
   1221 defm : avx512_binop_pat<andn, KANDNWrr>;
   1222 defm : avx512_binop_pat<or,   KORWrr>;
   1223 defm : avx512_binop_pat<xnor, KXNORWrr>;
   1224 defm : avx512_binop_pat<xor,  KXORWrr>;
   1225 
   1226 // Mask unpacking
   1227 multiclass avx512_mask_unpck<bits<8> opc, string OpcodeStr,
   1228                            RegisterClass KRC> {
   1229   let Predicates = [HasAVX512] in
   1230     def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2),
   1231                !strconcat(OpcodeStr,
   1232                           " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>;
   1233 }
   1234 
   1235 multiclass avx512_mask_unpck_bw<bits<8> opc, string OpcodeStr> {
   1236   defm BW : avx512_mask_unpck<opc, !strconcat(OpcodeStr, "bw"), VK16>,
   1237                             VEX_4V, VEX_L, PD;
   1238 }
   1239 
   1240 defm KUNPCK : avx512_mask_unpck_bw<0x4b, "kunpck">;
   1241 def : Pat<(v16i1 (concat_vectors (v8i1 VK8:$src1), (v8i1 VK8:$src2))),
   1242           (KUNPCKBWrr (COPY_TO_REGCLASS VK8:$src2, VK16),
   1243                   (COPY_TO_REGCLASS VK8:$src1, VK16))>;
   1244 
   1245 
   1246 multiclass avx512_mask_unpck_int<string IntName, string InstName> {
   1247   let Predicates = [HasAVX512] in
   1248     def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_bw")
   1249                 (i16 GR16:$src1), (i16 GR16:$src2)),
   1250               (COPY_TO_REGCLASS (!cast<Instruction>(InstName##"BWrr")
   1251               (v16i1 (COPY_TO_REGCLASS GR16:$src1, VK16)),
   1252               (v16i1 (COPY_TO_REGCLASS GR16:$src2, VK16))), GR16)>;
   1253 }
   1254 defm : avx512_mask_unpck_int<"kunpck",  "KUNPCK">;
   1255 
   1256 // Mask bit testing
   1257 multiclass avx512_mask_testop<bits<8> opc, string OpcodeStr, RegisterClass KRC,
   1258                             SDNode OpNode> {
   1259   let Predicates = [HasAVX512], Defs = [EFLAGS] in
   1260     def rr : I<opc, MRMSrcReg, (outs), (ins KRC:$src1, KRC:$src2),
   1261                !strconcat(OpcodeStr, " \t{$src2, $src1|$src1, $src2}"),
   1262                [(set EFLAGS, (OpNode KRC:$src1, KRC:$src2))]>;
   1263 }
   1264 
   1265 multiclass avx512_mask_testop_w<bits<8> opc, string OpcodeStr, SDNode OpNode> {
   1266   defm W : avx512_mask_testop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode>,
   1267                             VEX, PS;
   1268 }
   1269 
   1270 defm KORTEST : avx512_mask_testop_w<0x98, "kortest", X86kortest>;
   1271 
   1272 def : Pat<(X86cmp VK1:$src1, (i1 0)),
   1273           (KORTESTWrr (COPY_TO_REGCLASS VK1:$src1, VK16),
   1274            (COPY_TO_REGCLASS VK1:$src1, VK16))>;
   1275 
   1276 // Mask shift
   1277 multiclass avx512_mask_shiftop<bits<8> opc, string OpcodeStr, RegisterClass KRC,
   1278                              SDNode OpNode> {
   1279   let Predicates = [HasAVX512] in
   1280     def ri : Ii8<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src, i8imm:$imm),
   1281                  !strconcat(OpcodeStr,
   1282                             " \t{$imm, $src, $dst|$dst, $src, $imm}"),
   1283                             [(set KRC:$dst, (OpNode KRC:$src, (i8 imm:$imm)))]>;
   1284 }
   1285 
   1286 multiclass avx512_mask_shiftop_w<bits<8> opc1, bits<8> opc2, string OpcodeStr,
   1287                                SDNode OpNode> {
   1288   defm W : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "w"), VK16, OpNode>,
   1289                              VEX, TAPD, VEX_W;
   1290 }
   1291 
   1292 defm KSHIFTL : avx512_mask_shiftop_w<0x32, 0x33, "kshiftl", X86vshli>;
   1293 defm KSHIFTR : avx512_mask_shiftop_w<0x30, 0x31, "kshiftr", X86vsrli>;
   1294 
   1295 // Mask setting all 0s or 1s
   1296 multiclass avx512_mask_setop<RegisterClass KRC, ValueType VT, PatFrag Val> {
   1297   let Predicates = [HasAVX512] in
   1298     let isReMaterializable = 1, isAsCheapAsAMove = 1, isPseudo = 1 in
   1299       def #NAME# : I<0, Pseudo, (outs KRC:$dst), (ins), "",
   1300                      [(set KRC:$dst, (VT Val))]>;
   1301 }
   1302 
   1303 multiclass avx512_mask_setop_w<PatFrag Val> {
   1304   defm B : avx512_mask_setop<VK8,   v8i1, Val>;
   1305   defm W : avx512_mask_setop<VK16, v16i1, Val>;
   1306 }
   1307 
   1308 defm KSET0 : avx512_mask_setop_w<immAllZerosV>;
   1309 defm KSET1 : avx512_mask_setop_w<immAllOnesV>;
   1310 
   1311 // With AVX-512 only, 8-bit mask is promoted to 16-bit mask.
   1312 let Predicates = [HasAVX512] in {
   1313   def : Pat<(v8i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK8)>;
   1314   def : Pat<(v8i1 immAllOnesV),  (COPY_TO_REGCLASS (KSET1W), VK8)>;
   1315   def : Pat<(i1 0), (COPY_TO_REGCLASS (KSET0W), VK1)>;
   1316   def : Pat<(i1 1), (COPY_TO_REGCLASS (KSET1W), VK1)>;
   1317   def : Pat<(i1 -1), (COPY_TO_REGCLASS (KSET1W), VK1)>;
   1318 }
   1319 def : Pat<(v8i1 (extract_subvector (v16i1 VK16:$src), (iPTR 0))),
   1320           (v8i1 (COPY_TO_REGCLASS VK16:$src, VK8))>;
   1321 
   1322 def : Pat<(v16i1 (insert_subvector undef, (v8i1 VK8:$src), (iPTR 0))),
   1323           (v16i1 (COPY_TO_REGCLASS VK8:$src, VK16))>;
   1324 
   1325 def : Pat<(v8i1 (extract_subvector (v16i1 VK16:$src), (iPTR 8))),
   1326           (v8i1 (COPY_TO_REGCLASS (KSHIFTRWri VK16:$src, (i8 8)), VK8))>;
   1327 
   1328 def : Pat<(v8i1 (X86vshli VK8:$src, (i8 imm:$imm))),
   1329           (v8i1 (COPY_TO_REGCLASS (KSHIFTLWri (COPY_TO_REGCLASS VK8:$src, VK16), (I8Imm $imm)), VK8))>;
   1330 
   1331 def : Pat<(v8i1 (X86vsrli VK8:$src, (i8 imm:$imm))),
   1332           (v8i1 (COPY_TO_REGCLASS (KSHIFTRWri (COPY_TO_REGCLASS VK8:$src, VK16), (I8Imm $imm)), VK8))>;
   1333 //===----------------------------------------------------------------------===//
   1334 // AVX-512 - Aligned and unaligned load and store
   1335 //
   1336 
   1337 multiclass avx512_load<bits<8> opc, RegisterClass RC, RegisterClass KRC,
   1338                             X86MemOperand x86memop, PatFrag ld_frag, 
   1339                             string asm, Domain d,
   1340                             ValueType vt, bit IsReMaterializable = 1> {
   1341 let hasSideEffects = 0 in {
   1342   def rr : AVX512PI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src),
   1343               !strconcat(asm, " \t{$src, $dst|$dst, $src}"), [], d>,
   1344               EVEX;
   1345   def rrkz : AVX512PI<opc, MRMSrcReg, (outs RC:$dst), (ins KRC:$mask, RC:$src),
   1346                !strconcat(asm,
   1347                " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"),
   1348                [], d>, EVEX, EVEX_KZ;
   1349   }
   1350   let canFoldAsLoad = 1, isReMaterializable = IsReMaterializable in
   1351   def rm : AVX512PI<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src),
   1352               !strconcat(asm, " \t{$src, $dst|$dst, $src}"),
   1353                [(set (vt RC:$dst), (ld_frag addr:$src))], d>, EVEX;
   1354   let Constraints = "$src1 = $dst",  hasSideEffects = 0 in {
   1355   def rrk : AVX512PI<opc, MRMSrcReg, (outs RC:$dst), 
   1356                                      (ins RC:$src1, KRC:$mask, RC:$src2),
   1357               !strconcat(asm, 
   1358               " \t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), [], d>,
   1359               EVEX, EVEX_K;
   1360   let mayLoad = 1 in
   1361   def rmk : AVX512PI<opc, MRMSrcMem, (outs RC:$dst),
   1362                                 (ins RC:$src1, KRC:$mask, x86memop:$src2),
   1363               !strconcat(asm, 
   1364               " \t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"),
   1365                [], d>, EVEX, EVEX_K;
   1366   }
   1367   let mayLoad = 1 in
   1368   def rmkz : AVX512PI<opc, MRMSrcMem, (outs RC:$dst),
   1369                       (ins KRC:$mask, x86memop:$src2),
   1370               !strconcat(asm,
   1371               " \t{$src2, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src2}"),
   1372                [], d>, EVEX, EVEX_KZ;
   1373 }
   1374 
   1375 multiclass avx512_store<bits<8> opc, RegisterClass RC, RegisterClass KRC,
   1376                             X86MemOperand x86memop, PatFrag store_frag,
   1377                             string asm, Domain d, ValueType vt> {
   1378   let isAsmParserOnly = 1, hasSideEffects = 0 in {
   1379   def rr_alt : AVX512PI<opc, MRMDestReg, (outs RC:$dst), (ins RC:$src),
   1380               !strconcat(asm, " \t{$src, $dst|$dst, $src}"), [], d>,
   1381               EVEX;
   1382   let Constraints = "$src1 = $dst" in
   1383   def alt_rrk : AVX512PI<opc, MRMDestReg, (outs  RC:$dst),
   1384                                      (ins RC:$src1, KRC:$mask, RC:$src2),
   1385               !strconcat(asm,
   1386               " \t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), [], d>,
   1387               EVEX, EVEX_K;
   1388   def alt_rrkz : AVX512PI<opc, MRMDestReg, (outs  RC:$dst),
   1389                                            (ins KRC:$mask, RC:$src),
   1390               !strconcat(asm,
   1391               " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"),
   1392               [], d>, EVEX, EVEX_KZ;
   1393   }
   1394   let mayStore = 1 in {
   1395   def mr : AVX512PI<opc, MRMDestMem, (outs), (ins x86memop:$dst, RC:$src),
   1396               !strconcat(asm, " \t{$src, $dst|$dst, $src}"),
   1397                [(store_frag (vt RC:$src), addr:$dst)], d>, EVEX;
   1398   def mrk : AVX512PI<opc, MRMDestMem, (outs),
   1399                                 (ins x86memop:$dst, KRC:$mask, RC:$src),
   1400               !strconcat(asm,
   1401               " \t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}"),
   1402                [], d>, EVEX, EVEX_K;
   1403   def mrkz : AVX512PI<opc, MRMDestMem, (outs),
   1404                       (ins x86memop:$dst, KRC:$mask, RC:$src),
   1405               !strconcat(asm,
   1406               " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"),
   1407                [], d>, EVEX, EVEX_KZ;
   1408   }
   1409 }
   1410 
   1411 defm VMOVAPSZ : avx512_load<0x28, VR512, VK16WM, f512mem, alignedloadv16f32,
   1412                               "vmovaps", SSEPackedSingle, v16f32>,
   1413                 avx512_store<0x29, VR512, VK16WM, f512mem, alignedstore512,
   1414                               "vmovaps", SSEPackedSingle, v16f32>,
   1415                                PS, EVEX_V512, EVEX_CD8<32, CD8VF>;
   1416 defm VMOVAPDZ : avx512_load<0x28, VR512, VK8WM, f512mem, alignedloadv8f64,
   1417                               "vmovapd", SSEPackedDouble, v8f64>,
   1418                 avx512_store<0x29, VR512, VK8WM, f512mem, alignedstore512,
   1419                               "vmovapd", SSEPackedDouble, v8f64>,
   1420                               PD, EVEX_V512, VEX_W,
   1421                               EVEX_CD8<64, CD8VF>;
   1422 defm VMOVUPSZ : avx512_load<0x10, VR512, VK16WM, f512mem, loadv16f32,
   1423                               "vmovups", SSEPackedSingle, v16f32>,
   1424                 avx512_store<0x11, VR512, VK16WM, f512mem, store,
   1425                               "vmovups", SSEPackedSingle, v16f32>,
   1426                               PS, EVEX_V512, EVEX_CD8<32, CD8VF>;
   1427 defm VMOVUPDZ : avx512_load<0x10, VR512, VK8WM, f512mem, loadv8f64,
   1428                               "vmovupd", SSEPackedDouble, v8f64, 0>,
   1429                 avx512_store<0x11, VR512, VK8WM, f512mem, store,
   1430                               "vmovupd", SSEPackedDouble, v8f64>,
   1431                                PD, EVEX_V512, VEX_W,
   1432                                EVEX_CD8<64, CD8VF>;
   1433 def: Pat<(v8f64 (int_x86_avx512_mask_loadu_pd_512 addr:$ptr,
   1434                  (bc_v8f64 (v16i32 immAllZerosV)), GR8:$mask)),
   1435        (VMOVUPDZrmkz (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), addr:$ptr)>;
   1436 
   1437 def: Pat<(v16f32 (int_x86_avx512_mask_loadu_ps_512 addr:$ptr,
   1438                  (bc_v16f32 (v16i32 immAllZerosV)), GR16:$mask)),
   1439        (VMOVUPSZrmkz (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), addr:$ptr)>;
   1440 
   1441 def: Pat<(int_x86_avx512_mask_storeu_ps_512 addr:$ptr, (v16f32 VR512:$src),
   1442           GR16:$mask),
   1443          (VMOVUPSZmrk addr:$ptr, (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)),
   1444             VR512:$src)>;
   1445 def: Pat<(int_x86_avx512_mask_storeu_pd_512 addr:$ptr, (v8f64 VR512:$src),
   1446           GR8:$mask),
   1447          (VMOVUPDZmrk addr:$ptr, (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)),
   1448             VR512:$src)>;
   1449 
   1450 defm VMOVDQA32: avx512_load<0x6F, VR512, VK16WM, i512mem, alignedloadv16i32,
   1451                               "vmovdqa32", SSEPackedInt, v16i32>,
   1452                 avx512_store<0x7F, VR512, VK16WM, i512mem, alignedstore512,
   1453                               "vmovdqa32", SSEPackedInt, v16i32>,
   1454                                PD, EVEX_V512, EVEX_CD8<32, CD8VF>;
   1455 defm VMOVDQA64: avx512_load<0x6F, VR512, VK8WM, i512mem, alignedloadv8i64,
   1456                               "vmovdqa64", SSEPackedInt, v8i64>,
   1457                 avx512_store<0x7F, VR512, VK8WM, i512mem, alignedstore512,
   1458                               "vmovdqa64", SSEPackedInt, v8i64>,
   1459                                PD, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
   1460 defm VMOVDQU32: avx512_load<0x6F, VR512, VK16WM, i512mem, load,
   1461                               "vmovdqu32", SSEPackedInt, v16i32>,
   1462                 avx512_store<0x7F, VR512, VK16WM, i512mem, store,
   1463                               "vmovdqu32", SSEPackedInt, v16i32>,
   1464                                XS, EVEX_V512, EVEX_CD8<32, CD8VF>;
   1465 defm VMOVDQU64: avx512_load<0x6F, VR512, VK8WM, i512mem, load,
   1466                               "vmovdqu64", SSEPackedInt, v8i64>,
   1467                 avx512_store<0x7F, VR512, VK8WM, i512mem, store,
   1468                               "vmovdqu64", SSEPackedInt, v8i64>,
   1469                                XS, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
   1470 
   1471 def: Pat<(v16i32 (int_x86_avx512_mask_loadu_d_512 addr:$ptr,
   1472                  (v16i32 immAllZerosV), GR16:$mask)),
   1473        (VMOVDQU32rmkz (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), addr:$ptr)>;
   1474 
   1475 def: Pat<(v8i64 (int_x86_avx512_mask_loadu_q_512 addr:$ptr,
   1476                  (bc_v8i64 (v16i32 immAllZerosV)), GR8:$mask)),
   1477        (VMOVDQU64rmkz (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), addr:$ptr)>;
   1478 
   1479 def: Pat<(int_x86_avx512_mask_storeu_d_512 addr:$ptr, (v16i32 VR512:$src),
   1480           GR16:$mask),
   1481          (VMOVDQU32mrk addr:$ptr, (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)),
   1482             VR512:$src)>;
   1483 def: Pat<(int_x86_avx512_mask_storeu_q_512 addr:$ptr, (v8i64 VR512:$src),
   1484           GR8:$mask),
   1485          (VMOVDQU64mrk addr:$ptr, (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)),
   1486             VR512:$src)>;
   1487 
   1488 let AddedComplexity = 20 in {
   1489 def : Pat<(v8i64 (vselect VK8WM:$mask, (v8i64 VR512:$src),
   1490                            (bc_v8i64 (v16i32 immAllZerosV)))),
   1491                   (VMOVDQU64rrkz VK8WM:$mask, VR512:$src)>;
   1492 
   1493 def : Pat<(v8i64 (vselect VK8WM:$mask, (bc_v8i64 (v16i32 immAllZerosV)),
   1494                   (v8i64 VR512:$src))),
   1495    (VMOVDQU64rrkz (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$mask, VK16)),
   1496                                               VK8), VR512:$src)>;
   1497 
   1498 def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 VR512:$src),
   1499                            (v16i32 immAllZerosV))),
   1500                   (VMOVDQU32rrkz VK16WM:$mask, VR512:$src)>;
   1501 
   1502 def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 immAllZerosV),
   1503                    (v16i32 VR512:$src))),
   1504    (VMOVDQU32rrkz (KNOTWrr VK16WM:$mask), VR512:$src)>;
   1505                                               
   1506 def : Pat<(v16f32 (vselect VK16WM:$mask, (v16f32 VR512:$src1), 
   1507                            (v16f32 VR512:$src2))),
   1508                   (VMOVUPSZrrk VR512:$src2, VK16WM:$mask, VR512:$src1)>;
   1509 def : Pat<(v8f64 (vselect VK8WM:$mask, (v8f64 VR512:$src1), 
   1510                            (v8f64 VR512:$src2))),
   1511                   (VMOVUPDZrrk VR512:$src2, VK8WM:$mask, VR512:$src1)>;
   1512 def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 VR512:$src1), 
   1513                            (v16i32 VR512:$src2))),
   1514                   (VMOVDQU32rrk VR512:$src2, VK16WM:$mask, VR512:$src1)>;
   1515 def : Pat<(v8i64 (vselect VK8WM:$mask, (v8i64 VR512:$src1), 
   1516                            (v8i64 VR512:$src2))),
   1517                   (VMOVDQU64rrk VR512:$src2, VK8WM:$mask, VR512:$src1)>;
   1518 }
   1519 // Move Int Doubleword to Packed Double Int
   1520 //
   1521 def VMOVDI2PDIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR32:$src),
   1522                       "vmovd\t{$src, $dst|$dst, $src}",
   1523                       [(set VR128X:$dst,
   1524                         (v4i32 (scalar_to_vector GR32:$src)))], IIC_SSE_MOVDQ>,
   1525                         EVEX, VEX_LIG;
   1526 def VMOVDI2PDIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst), (ins i32mem:$src),
   1527                       "vmovd\t{$src, $dst|$dst, $src}",
   1528                       [(set VR128X:$dst,
   1529                         (v4i32 (scalar_to_vector (loadi32 addr:$src))))],
   1530                         IIC_SSE_MOVDQ>, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>;
   1531 def VMOV64toPQIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR64:$src),
   1532                       "vmovq\t{$src, $dst|$dst, $src}",
   1533                         [(set VR128X:$dst,
   1534                           (v2i64 (scalar_to_vector GR64:$src)))],
   1535                           IIC_SSE_MOVDQ>, EVEX, VEX_W, VEX_LIG;
   1536 let isCodeGenOnly = 1 in {
   1537 def VMOV64toSDZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR64:$dst), (ins GR64:$src),
   1538                        "vmovq\t{$src, $dst|$dst, $src}",
   1539                        [(set FR64:$dst, (bitconvert GR64:$src))],
   1540                        IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteMove]>;
   1541 def VMOVSDto64Zrr : AVX512BI<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64:$src),
   1542                          "vmovq\t{$src, $dst|$dst, $src}",
   1543                          [(set GR64:$dst, (bitconvert FR64:$src))],
   1544                          IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteMove]>;
   1545 }
   1546 def VMOVSDto64Zmr : AVX512BI<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64:$src),
   1547                          "vmovq\t{$src, $dst|$dst, $src}",
   1548                          [(store (i64 (bitconvert FR64:$src)), addr:$dst)],
   1549                          IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteStore]>,
   1550                          EVEX_CD8<64, CD8VT1>;
   1551 
   1552 // Move Int Doubleword to Single Scalar
   1553 //
   1554 let isCodeGenOnly = 1 in {
   1555 def VMOVDI2SSZrr  : AVX512BI<0x6E, MRMSrcReg, (outs FR32X:$dst), (ins GR32:$src),
   1556                       "vmovd\t{$src, $dst|$dst, $src}",
   1557                       [(set FR32X:$dst, (bitconvert GR32:$src))],
   1558                       IIC_SSE_MOVDQ>, EVEX, VEX_LIG;
   1559 
   1560 def VMOVDI2SSZrm  : AVX512BI<0x6E, MRMSrcMem, (outs FR32X:$dst), (ins i32mem:$src),
   1561                       "vmovd\t{$src, $dst|$dst, $src}",
   1562                       [(set FR32X:$dst, (bitconvert (loadi32 addr:$src)))],
   1563                       IIC_SSE_MOVDQ>, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>;
   1564 }
   1565 
   1566 // Move doubleword from xmm register to r/m32
   1567 //
   1568 def VMOVPDI2DIZrr  : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128X:$src),
   1569                        "vmovd\t{$src, $dst|$dst, $src}",
   1570                        [(set GR32:$dst, (vector_extract (v4i32 VR128X:$src),
   1571                                         (iPTR 0)))], IIC_SSE_MOVD_ToGP>,
   1572                        EVEX, VEX_LIG;
   1573 def VMOVPDI2DIZmr  : AVX512BI<0x7E, MRMDestMem, (outs),
   1574                        (ins i32mem:$dst, VR128X:$src),
   1575                        "vmovd\t{$src, $dst|$dst, $src}",
   1576                        [(store (i32 (vector_extract (v4i32 VR128X:$src),
   1577                                      (iPTR 0))), addr:$dst)], IIC_SSE_MOVDQ>,
   1578                        EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>;
   1579 
   1580 // Move quadword from xmm1 register to r/m64
   1581 //
   1582 def VMOVPQIto64Zrr : I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128X:$src),
   1583                       "vmovq\t{$src, $dst|$dst, $src}",
   1584                       [(set GR64:$dst, (extractelt (v2i64 VR128X:$src),
   1585                                                    (iPTR 0)))],
   1586                       IIC_SSE_MOVD_ToGP>, PD, EVEX, VEX_LIG, VEX_W,
   1587                       Requires<[HasAVX512, In64BitMode]>;
   1588 
   1589 def VMOVPQIto64Zmr : I<0xD6, MRMDestMem, (outs),
   1590                        (ins i64mem:$dst, VR128X:$src),
   1591                        "vmovq\t{$src, $dst|$dst, $src}",
   1592                        [(store (extractelt (v2i64 VR128X:$src), (iPTR 0)),
   1593                                addr:$dst)], IIC_SSE_MOVDQ>,
   1594                        EVEX, PD, VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>,
   1595                        Sched<[WriteStore]>, Requires<[HasAVX512, In64BitMode]>;
   1596 
   1597 // Move Scalar Single to Double Int
   1598 //
   1599 let isCodeGenOnly = 1 in {
   1600 def VMOVSS2DIZrr  : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst),
   1601                       (ins FR32X:$src),
   1602                       "vmovd\t{$src, $dst|$dst, $src}",
   1603                       [(set GR32:$dst, (bitconvert FR32X:$src))],
   1604                       IIC_SSE_MOVD_ToGP>, EVEX, VEX_LIG;
   1605 def VMOVSS2DIZmr  : AVX512BI<0x7E, MRMDestMem, (outs),
   1606                       (ins i32mem:$dst, FR32X:$src),
   1607                       "vmovd\t{$src, $dst|$dst, $src}",
   1608                       [(store (i32 (bitconvert FR32X:$src)), addr:$dst)],
   1609                       IIC_SSE_MOVDQ>, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>;
   1610 }
   1611 
   1612 // Move Quadword Int to Packed Quadword Int
   1613 //
   1614 def VMOVQI2PQIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst),
   1615                       (ins i64mem:$src),
   1616                       "vmovq\t{$src, $dst|$dst, $src}",
   1617                       [(set VR128X:$dst,
   1618                         (v2i64 (scalar_to_vector (loadi64 addr:$src))))]>,
   1619                       EVEX, VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
   1620 
   1621 //===----------------------------------------------------------------------===//
   1622 // AVX-512  MOVSS, MOVSD
   1623 //===----------------------------------------------------------------------===//
   1624 
   1625 multiclass avx512_move_scalar <string asm, RegisterClass RC, 
   1626                               SDNode OpNode, ValueType vt,
   1627                               X86MemOperand x86memop, PatFrag mem_pat> {
   1628   let hasSideEffects = 0 in {
   1629   def rr : SI<0x10, MRMSrcReg, (outs VR128X:$dst), (ins VR128X:$src1, RC:$src2), 
   1630               !strconcat(asm, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   1631               [(set VR128X:$dst, (vt (OpNode VR128X:$src1,
   1632                                       (scalar_to_vector RC:$src2))))],
   1633               IIC_SSE_MOV_S_RR>, EVEX_4V, VEX_LIG;
   1634   let Constraints = "$src1 = $dst" in
   1635   def rrk : SI<0x10, MRMSrcReg, (outs VR128X:$dst),
   1636               (ins VR128X:$src1, VK1WM:$mask, RC:$src2, RC:$src3),
   1637               !strconcat(asm,
   1638                 " \t{$src3, $src2, $dst {${mask}}|$dst {${mask}}, $src2, $src3}"),
   1639               [], IIC_SSE_MOV_S_RR>, EVEX_4V, VEX_LIG, EVEX_K;
   1640   def rm : SI<0x10, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src),
   1641               !strconcat(asm, " \t{$src, $dst|$dst, $src}"),
   1642               [(set RC:$dst, (mem_pat addr:$src))], IIC_SSE_MOV_S_RM>,
   1643               EVEX, VEX_LIG;
   1644   def mr: SI<0x11, MRMDestMem, (outs), (ins x86memop:$dst, RC:$src),
   1645              !strconcat(asm, " \t{$src, $dst|$dst, $src}"),
   1646              [(store RC:$src, addr:$dst)], IIC_SSE_MOV_S_MR>,
   1647              EVEX, VEX_LIG;
   1648   } //hasSideEffects = 0
   1649 }
   1650 
   1651 let ExeDomain = SSEPackedSingle in
   1652 defm VMOVSSZ : avx512_move_scalar<"movss", FR32X, X86Movss, v4f32, f32mem,
   1653                                  loadf32>, XS, EVEX_CD8<32, CD8VT1>;
   1654 
   1655 let ExeDomain = SSEPackedDouble in
   1656 defm VMOVSDZ : avx512_move_scalar<"movsd", FR64X, X86Movsd, v2f64, f64mem,
   1657                                  loadf64>, XD, VEX_W, EVEX_CD8<64, CD8VT1>;
   1658 
   1659 def : Pat<(f32 (X86select VK1WM:$mask, (f32 FR32X:$src1), (f32 FR32X:$src2))),
   1660           (COPY_TO_REGCLASS (VMOVSSZrrk (COPY_TO_REGCLASS FR32X:$src2, VR128X),
   1661            VK1WM:$mask, (f32 (IMPLICIT_DEF)), FR32X:$src1), FR32X)>;
   1662 
   1663 def : Pat<(f64 (X86select VK1WM:$mask, (f64 FR64X:$src1), (f64 FR64X:$src2))),
   1664           (COPY_TO_REGCLASS (VMOVSDZrrk (COPY_TO_REGCLASS FR64X:$src2, VR128X),
   1665            VK1WM:$mask, (f64 (IMPLICIT_DEF)), FR64X:$src1), FR64X)>;
   1666 
   1667 // For the disassembler
   1668 let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in {
   1669   def VMOVSSZrr_REV : SI<0x11, MRMDestReg, (outs VR128X:$dst),
   1670                         (ins VR128X:$src1, FR32X:$src2),
   1671                         "movss\t{$src2, $src1, $dst|$dst, $src1, $src2}", [],
   1672                         IIC_SSE_MOV_S_RR>,
   1673                         XS, EVEX_4V, VEX_LIG;
   1674   def VMOVSDZrr_REV : SI<0x11, MRMDestReg, (outs VR128X:$dst),
   1675                         (ins VR128X:$src1, FR64X:$src2),
   1676                         "movsd\t{$src2, $src1, $dst|$dst, $src1, $src2}", [],
   1677                         IIC_SSE_MOV_S_RR>,
   1678                         XD, EVEX_4V, VEX_LIG, VEX_W;
   1679 }
   1680 
   1681 let Predicates = [HasAVX512] in {
   1682   let AddedComplexity = 15 in {
   1683   // Move scalar to XMM zero-extended, zeroing a VR128X then do a
   1684   // MOVS{S,D} to the lower bits.
   1685   def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector FR32X:$src)))),
   1686             (VMOVSSZrr (v4f32 (V_SET0)), FR32X:$src)>;
   1687   def : Pat<(v4f32 (X86vzmovl (v4f32 VR128X:$src))),
   1688             (VMOVSSZrr (v4f32 (V_SET0)), (COPY_TO_REGCLASS VR128X:$src, FR32X))>;
   1689   def : Pat<(v4i32 (X86vzmovl (v4i32 VR128X:$src))),
   1690             (VMOVSSZrr (v4i32 (V_SET0)), (COPY_TO_REGCLASS VR128X:$src, FR32X))>;
   1691   def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector FR64X:$src)))),
   1692             (VMOVSDZrr (v2f64 (V_SET0)), FR64X:$src)>;
   1693 
   1694   // Move low f32 and clear high bits.
   1695   def : Pat<(v8f32 (X86vzmovl (v8f32 VR256X:$src))),
   1696             (SUBREG_TO_REG (i32 0),
   1697              (VMOVSSZrr (v4f32 (V_SET0)), 
   1698               (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm)), sub_xmm)>;
   1699   def : Pat<(v8i32 (X86vzmovl (v8i32 VR256X:$src))),
   1700             (SUBREG_TO_REG (i32 0),
   1701              (VMOVSSZrr (v4i32 (V_SET0)),
   1702                        (EXTRACT_SUBREG (v8i32 VR256X:$src), sub_xmm)), sub_xmm)>;
   1703   }
   1704 
   1705   let AddedComplexity = 20 in {
   1706   // MOVSSrm zeros the high parts of the register; represent this
   1707   // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0
   1708   def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector (loadf32 addr:$src))))),
   1709             (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>;
   1710   def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))),
   1711             (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>;
   1712   def : Pat<(v4f32 (X86vzmovl (loadv4f32 addr:$src))),
   1713             (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>;
   1714 
   1715   // MOVSDrm zeros the high parts of the register; represent this
   1716   // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0
   1717   def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector (loadf64 addr:$src))))),
   1718             (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
   1719   def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))),
   1720             (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
   1721   def : Pat<(v2f64 (X86vzmovl (loadv2f64 addr:$src))),
   1722             (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
   1723   def : Pat<(v2f64 (X86vzmovl (bc_v2f64 (loadv4f32 addr:$src)))),
   1724             (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
   1725   def : Pat<(v2f64 (X86vzload addr:$src)),
   1726             (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
   1727 
   1728   // Represent the same patterns above but in the form they appear for
   1729   // 256-bit types
   1730   def : Pat<(v8i32 (X86vzmovl (insert_subvector undef,
   1731                    (v4i32 (scalar_to_vector (loadi32 addr:$src))), (iPTR 0)))),
   1732             (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrm addr:$src), sub_xmm)>;
   1733   def : Pat<(v8f32 (X86vzmovl (insert_subvector undef,
   1734                    (v4f32 (scalar_to_vector (loadf32 addr:$src))), (iPTR 0)))),
   1735             (SUBREG_TO_REG (i32 0), (VMOVSSZrm addr:$src), sub_xmm)>;
   1736   def : Pat<(v4f64 (X86vzmovl (insert_subvector undef,
   1737                    (v2f64 (scalar_to_vector (loadf64 addr:$src))), (iPTR 0)))),
   1738             (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>;
   1739   }
   1740   def : Pat<(v8f32 (X86vzmovl (insert_subvector undef,
   1741                    (v4f32 (scalar_to_vector FR32X:$src)), (iPTR 0)))),
   1742             (SUBREG_TO_REG (i32 0), (v4f32 (VMOVSSZrr (v4f32 (V_SET0)),
   1743                                             FR32X:$src)), sub_xmm)>;
   1744   def : Pat<(v4f64 (X86vzmovl (insert_subvector undef,
   1745                    (v2f64 (scalar_to_vector FR64X:$src)), (iPTR 0)))),
   1746             (SUBREG_TO_REG (i64 0), (v2f64 (VMOVSDZrr (v2f64 (V_SET0)),
   1747                                      FR64X:$src)), sub_xmm)>;
   1748   def : Pat<(v4i64 (X86vzmovl (insert_subvector undef,
   1749                    (v2i64 (scalar_to_vector (loadi64 addr:$src))), (iPTR 0)))),
   1750             (SUBREG_TO_REG (i64 0), (VMOVQI2PQIZrm addr:$src), sub_xmm)>;
   1751 
   1752   // Move low f64 and clear high bits.
   1753   def : Pat<(v4f64 (X86vzmovl (v4f64 VR256X:$src))),
   1754             (SUBREG_TO_REG (i32 0),
   1755              (VMOVSDZrr (v2f64 (V_SET0)),
   1756                        (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm)), sub_xmm)>;
   1757 
   1758   def : Pat<(v4i64 (X86vzmovl (v4i64 VR256X:$src))),
   1759             (SUBREG_TO_REG (i32 0), (VMOVSDZrr (v2i64 (V_SET0)),
   1760                        (EXTRACT_SUBREG (v4i64 VR256X:$src), sub_xmm)), sub_xmm)>;
   1761 
   1762   // Extract and store.
   1763   def : Pat<(store (f32 (vector_extract (v4f32 VR128X:$src), (iPTR 0))),
   1764                    addr:$dst),
   1765             (VMOVSSZmr addr:$dst, (COPY_TO_REGCLASS (v4f32 VR128X:$src), FR32X))>;
   1766   def : Pat<(store (f64 (vector_extract (v2f64 VR128X:$src), (iPTR 0))),
   1767                    addr:$dst),
   1768             (VMOVSDZmr addr:$dst, (COPY_TO_REGCLASS (v2f64 VR128X:$src), FR64X))>;
   1769 
   1770   // Shuffle with VMOVSS
   1771   def : Pat<(v4i32 (X86Movss VR128X:$src1, VR128X:$src2)),
   1772             (VMOVSSZrr (v4i32 VR128X:$src1),
   1773                       (COPY_TO_REGCLASS (v4i32 VR128X:$src2), FR32X))>;
   1774   def : Pat<(v4f32 (X86Movss VR128X:$src1, VR128X:$src2)),
   1775             (VMOVSSZrr (v4f32 VR128X:$src1),
   1776                       (COPY_TO_REGCLASS (v4f32 VR128X:$src2), FR32X))>;
   1777 
   1778   // 256-bit variants
   1779   def : Pat<(v8i32 (X86Movss VR256X:$src1, VR256X:$src2)),
   1780             (SUBREG_TO_REG (i32 0),
   1781               (VMOVSSZrr (EXTRACT_SUBREG (v8i32 VR256X:$src1), sub_xmm),
   1782                         (EXTRACT_SUBREG (v8i32 VR256X:$src2), sub_xmm)),
   1783               sub_xmm)>;
   1784   def : Pat<(v8f32 (X86Movss VR256X:$src1, VR256X:$src2)),
   1785             (SUBREG_TO_REG (i32 0),
   1786               (VMOVSSZrr (EXTRACT_SUBREG (v8f32 VR256X:$src1), sub_xmm),
   1787                         (EXTRACT_SUBREG (v8f32 VR256X:$src2), sub_xmm)),
   1788               sub_xmm)>;
   1789 
   1790   // Shuffle with VMOVSD
   1791   def : Pat<(v2i64 (X86Movsd VR128X:$src1, VR128X:$src2)),
   1792             (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
   1793   def : Pat<(v2f64 (X86Movsd VR128X:$src1, VR128X:$src2)),
   1794             (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
   1795   def : Pat<(v4f32 (X86Movsd VR128X:$src1, VR128X:$src2)),
   1796             (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
   1797   def : Pat<(v4i32 (X86Movsd VR128X:$src1, VR128X:$src2)),
   1798             (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
   1799 
   1800   // 256-bit variants
   1801   def : Pat<(v4i64 (X86Movsd VR256X:$src1, VR256X:$src2)),
   1802             (SUBREG_TO_REG (i32 0),
   1803               (VMOVSDZrr (EXTRACT_SUBREG (v4i64 VR256X:$src1), sub_xmm),
   1804                         (EXTRACT_SUBREG (v4i64 VR256X:$src2), sub_xmm)),
   1805               sub_xmm)>;
   1806   def : Pat<(v4f64 (X86Movsd VR256X:$src1, VR256X:$src2)),
   1807             (SUBREG_TO_REG (i32 0),
   1808               (VMOVSDZrr (EXTRACT_SUBREG (v4f64 VR256X:$src1), sub_xmm),
   1809                         (EXTRACT_SUBREG (v4f64 VR256X:$src2), sub_xmm)),
   1810               sub_xmm)>;
   1811 
   1812   def : Pat<(v2f64 (X86Movlpd VR128X:$src1, VR128X:$src2)),
   1813             (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
   1814   def : Pat<(v2i64 (X86Movlpd VR128X:$src1, VR128X:$src2)),
   1815             (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
   1816   def : Pat<(v4f32 (X86Movlps VR128X:$src1, VR128X:$src2)),
   1817             (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
   1818   def : Pat<(v4i32 (X86Movlps VR128X:$src1, VR128X:$src2)),
   1819             (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>;
   1820 }
   1821 
   1822 let AddedComplexity = 15 in
   1823 def VMOVZPQILo2PQIZrr : AVX512XSI<0x7E, MRMSrcReg, (outs VR128X:$dst),
   1824                                 (ins VR128X:$src),
   1825                                 "vmovq\t{$src, $dst|$dst, $src}",
   1826                                 [(set VR128X:$dst, (v2i64 (X86vzmovl 
   1827                                                    (v2i64 VR128X:$src))))],
   1828                                 IIC_SSE_MOVQ_RR>, EVEX, VEX_W;
   1829 
   1830 let AddedComplexity = 20 in
   1831 def VMOVZPQILo2PQIZrm : AVX512XSI<0x7E, MRMSrcMem, (outs VR128X:$dst),
   1832                                  (ins i128mem:$src),
   1833                                  "vmovq\t{$src, $dst|$dst, $src}",
   1834                                  [(set VR128X:$dst, (v2i64 (X86vzmovl
   1835                                                      (loadv2i64 addr:$src))))],
   1836                                  IIC_SSE_MOVDQ>, EVEX, VEX_W,
   1837                                  EVEX_CD8<8, CD8VT8>;
   1838 
   1839 let Predicates = [HasAVX512] in {
   1840   // AVX 128-bit movd/movq instruction write zeros in the high 128-bit part.
   1841   let AddedComplexity = 20 in {
   1842     def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector (loadi32 addr:$src))))),
   1843               (VMOVDI2PDIZrm addr:$src)>;
   1844     def : Pat<(v2i64 (X86vzmovl (v2i64 (scalar_to_vector GR64:$src)))),
   1845               (VMOV64toPQIZrr GR64:$src)>;
   1846     def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector GR32:$src)))),
   1847               (VMOVDI2PDIZrr GR32:$src)>;
   1848               
   1849     def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv4f32 addr:$src)))),
   1850               (VMOVDI2PDIZrm addr:$src)>;
   1851     def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv2i64 addr:$src)))),
   1852               (VMOVDI2PDIZrm addr:$src)>;
   1853     def : Pat<(v2i64 (X86vzmovl (loadv2i64 addr:$src))),
   1854             (VMOVZPQILo2PQIZrm addr:$src)>;
   1855     def : Pat<(v2f64 (X86vzmovl (v2f64 VR128X:$src))),
   1856             (VMOVZPQILo2PQIZrr VR128X:$src)>;
   1857     def : Pat<(v2i64 (X86vzload addr:$src)),
   1858             (VMOVZPQILo2PQIZrm addr:$src)>;
   1859   }
   1860 
   1861   // Use regular 128-bit instructions to match 256-bit scalar_to_vec+zext.
   1862   def : Pat<(v8i32 (X86vzmovl (insert_subvector undef,
   1863                                (v4i32 (scalar_to_vector GR32:$src)),(iPTR 0)))),
   1864             (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src), sub_xmm)>;
   1865   def : Pat<(v4i64 (X86vzmovl (insert_subvector undef,
   1866                                (v2i64 (scalar_to_vector GR64:$src)),(iPTR 0)))),
   1867             (SUBREG_TO_REG (i64 0), (VMOV64toPQIZrr GR64:$src), sub_xmm)>;
   1868 }
   1869 
   1870 def : Pat<(v16i32 (X86Vinsert (v16i32 immAllZerosV), GR32:$src2, (iPTR 0))),
   1871         (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src2), sub_xmm)>;
   1872 
   1873 def : Pat<(v8i64 (X86Vinsert (bc_v8i64 (v16i32 immAllZerosV)), GR64:$src2, (iPTR 0))),
   1874         (SUBREG_TO_REG (i32 0), (VMOV64toPQIZrr GR64:$src2), sub_xmm)>;
   1875 
   1876 def : Pat<(v16i32 (X86Vinsert undef, GR32:$src2, (iPTR 0))),
   1877         (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src2), sub_xmm)>;
   1878 
   1879 def : Pat<(v8i64 (X86Vinsert undef, GR64:$src2, (iPTR 0))),
   1880         (SUBREG_TO_REG (i32 0), (VMOV64toPQIZrr GR64:$src2), sub_xmm)>;
   1881 
   1882 //===----------------------------------------------------------------------===//
   1883 // AVX-512 - Non-temporals
   1884 //===----------------------------------------------------------------------===//
   1885 
   1886 def VMOVNTDQAZrm : AVX5128I<0x2A, MRMSrcMem, (outs VR512:$dst),
   1887                             (ins i512mem:$src),
   1888                             "vmovntdqa\t{$src, $dst|$dst, $src}",
   1889                             [(set VR512:$dst,
   1890                               (int_x86_avx512_movntdqa addr:$src))]>,
   1891                    EVEX, EVEX_V512, EVEX_CD8<64, CD8VF>;
   1892 
   1893 // Prefer non-temporal over temporal versions
   1894 let AddedComplexity = 400, SchedRW = [WriteStore] in {
   1895 
   1896 def VMOVNTPSZmr : AVX512PSI<0x2B, MRMDestMem, (outs),
   1897                             (ins f512mem:$dst, VR512:$src),
   1898                             "vmovntps\t{$src, $dst|$dst, $src}",
   1899                             [(alignednontemporalstore (v16f32 VR512:$src),
   1900                                                       addr:$dst)],
   1901                             IIC_SSE_MOVNT>,
   1902                   EVEX, EVEX_V512, EVEX_CD8<32, CD8VF>;
   1903 
   1904 def VMOVNTPDZmr : AVX512PDI<0x2B, MRMDestMem, (outs),
   1905                             (ins f512mem:$dst, VR512:$src),
   1906                             "vmovntpd\t{$src, $dst|$dst, $src}",
   1907                             [(alignednontemporalstore (v8f64 VR512:$src),
   1908                                                       addr:$dst)],
   1909 			    IIC_SSE_MOVNT>,
   1910                   EVEX, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
   1911 
   1912 
   1913 def VMOVNTDQZmr : AVX512BI<0xE7, MRMDestMem, (outs),
   1914                            (ins i512mem:$dst, VR512:$src),
   1915                            "vmovntdq\t{$src, $dst|$dst, $src}",
   1916                            [(alignednontemporalstore (v8i64 VR512:$src),
   1917                                                      addr:$dst)],
   1918                            IIC_SSE_MOVNT>,
   1919                   EVEX, EVEX_V512, EVEX_CD8<64, CD8VF>;
   1920 }
   1921 
   1922 //===----------------------------------------------------------------------===//
   1923 // AVX-512 - Integer arithmetic
   1924 //
   1925 multiclass avx512_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
   1926                         ValueType OpVT, RegisterClass KRC,
   1927                         RegisterClass RC, PatFrag memop_frag,
   1928                         X86MemOperand x86memop, PatFrag scalar_mfrag,
   1929                         X86MemOperand x86scalar_mop, string BrdcstStr,
   1930                         OpndItins itins, bit IsCommutable = 0> {
   1931   let isCommutable = IsCommutable in
   1932     def rr : AVX512BI<opc, MRMSrcReg, (outs RC:$dst),
   1933               (ins RC:$src1, RC:$src2),
   1934               !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   1935               [(set RC:$dst, (OpVT (OpNode (OpVT RC:$src1), (OpVT RC:$src2))))],
   1936               itins.rr>, EVEX_4V;
   1937   let AddedComplexity = 30 in {
   1938     let Constraints = "$src0 = $dst" in
   1939       def rrk : AVX512BI<opc, MRMSrcReg, (outs RC:$dst),
   1940                  (ins RC:$src0, KRC:$mask, RC:$src1, RC:$src2),
   1941                  !strconcat(OpcodeStr,
   1942                     " \t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"),
   1943                  [(set RC:$dst, (OpVT (vselect KRC:$mask,
   1944                                   (OpNode (OpVT RC:$src1), (OpVT RC:$src2)),
   1945                                   RC:$src0)))],
   1946                  itins.rr>, EVEX_4V, EVEX_K;
   1947     def rrkz : AVX512BI<opc, MRMSrcReg, (outs RC:$dst),
   1948                 (ins KRC:$mask, RC:$src1, RC:$src2),
   1949                 !strconcat(OpcodeStr, " \t{$src2, $src1, $dst {${mask}} {z}" ,
   1950                     "|$dst {${mask}} {z}, $src1, $src2}"),
   1951                 [(set RC:$dst, (OpVT (vselect KRC:$mask,
   1952                                   (OpNode (OpVT RC:$src1), (OpVT RC:$src2)),
   1953                                   (OpVT immAllZerosV))))],
   1954                 itins.rr>, EVEX_4V, EVEX_KZ;
   1955   }
   1956 
   1957   let mayLoad = 1 in {
   1958     def rm : AVX512BI<opc, MRMSrcMem, (outs RC:$dst),
   1959               (ins RC:$src1, x86memop:$src2),
   1960               !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   1961               [(set RC:$dst, (OpVT (OpNode (OpVT RC:$src1), (memop_frag addr:$src2))))],
   1962               itins.rm>, EVEX_4V;
   1963     let AddedComplexity = 30 in {
   1964     let Constraints = "$src0 = $dst" in
   1965       def rmk : AVX512BI<opc, MRMSrcMem, (outs RC:$dst),
   1966                  (ins RC:$src0, KRC:$mask, RC:$src1, x86memop:$src2),
   1967                  !strconcat(OpcodeStr,
   1968                      " \t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"),
   1969                  [(set RC:$dst, (OpVT (vselect KRC:$mask,
   1970                                     (OpNode (OpVT RC:$src1), (memop_frag addr:$src2)),
   1971                                     RC:$src0)))],
   1972                  itins.rm>, EVEX_4V, EVEX_K;
   1973     def rmkz : AVX512BI<opc, MRMSrcMem, (outs RC:$dst),
   1974                 (ins KRC:$mask, RC:$src1, x86memop:$src2),
   1975                 !strconcat(OpcodeStr,
   1976                     " \t{$src2, $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, $src2}"),
   1977                 [(set RC:$dst, (OpVT (vselect KRC:$mask,
   1978                                     (OpNode (OpVT RC:$src1), (memop_frag addr:$src2)),
   1979                                     (OpVT immAllZerosV))))],
   1980                 itins.rm>, EVEX_4V, EVEX_KZ;
   1981     }
   1982     def rmb : AVX512BI<opc, MRMSrcMem, (outs RC:$dst),
   1983                (ins RC:$src1, x86scalar_mop:$src2),
   1984                !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr,
   1985                           ", $src1, $dst|$dst, $src1, ${src2}", BrdcstStr, "}"),
   1986                [(set RC:$dst, (OpNode RC:$src1,
   1987                                (OpVT (X86VBroadcast (scalar_mfrag addr:$src2)))))],
   1988                itins.rm>, EVEX_4V, EVEX_B;
   1989     let AddedComplexity = 30 in {
   1990     let Constraints = "$src0 = $dst" in
   1991       def rmbk : AVX512BI<opc, MRMSrcMem, (outs RC:$dst),
   1992                   (ins RC:$src0, KRC:$mask, RC:$src1, x86scalar_mop:$src2),
   1993                   !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr,
   1994                              ", $src1, $dst {${mask}}|$dst {${mask}}, $src1, ${src2}",
   1995                              BrdcstStr, "}"),
   1996                   [(set RC:$dst, (OpVT (vselect KRC:$mask,
   1997                                     (OpNode (OpVT RC:$src1),
   1998                                      (OpVT (X86VBroadcast (scalar_mfrag addr:$src2)))),
   1999                                     RC:$src0)))],
   2000                   itins.rm>, EVEX_4V, EVEX_B, EVEX_K;
   2001     def rmbkz : AVX512BI<opc, MRMSrcMem, (outs RC:$dst),
   2002                  (ins KRC:$mask, RC:$src1, x86scalar_mop:$src2),
   2003                  !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr,
   2004                             ", $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, ${src2}",
   2005                             BrdcstStr, "}"),
   2006                  [(set RC:$dst, (OpVT (vselect KRC:$mask,
   2007                                     (OpNode (OpVT RC:$src1),
   2008                                      (OpVT (X86VBroadcast (scalar_mfrag addr:$src2)))),
   2009                                     (OpVT immAllZerosV))))],
   2010                  itins.rm>, EVEX_4V, EVEX_B, EVEX_KZ;
   2011     }
   2012   }
   2013 }
   2014 
   2015 multiclass avx512_binop_rm2<bits<8> opc, string OpcodeStr, ValueType DstVT,
   2016                             ValueType SrcVT, RegisterClass KRC, RegisterClass RC,
   2017                             PatFrag memop_frag, X86MemOperand x86memop,
   2018                             PatFrag scalar_mfrag, X86MemOperand x86scalar_mop,
   2019                             string BrdcstStr, OpndItins itins, bit IsCommutable = 0> {
   2020   let isCommutable = IsCommutable in
   2021   {
   2022     def rr : AVX512BI<opc, MRMSrcReg, (outs RC:$dst),
   2023        (ins RC:$src1, RC:$src2),
   2024        !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   2025        []>, EVEX_4V;
   2026     def rrk : AVX512BI<opc, MRMSrcReg, (outs RC:$dst),
   2027                (ins KRC:$mask, RC:$src1, RC:$src2),
   2028                !strconcat(OpcodeStr,
   2029                   " \t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"),
   2030                [], itins.rr>, EVEX_4V, EVEX_K;
   2031     def rrkz : AVX512BI<opc, MRMSrcReg, (outs RC:$dst),
   2032                 (ins KRC:$mask, RC:$src1, RC:$src2),
   2033                 !strconcat(OpcodeStr, " \t{$src2, $src1, $dst {${mask}} {z}" ,
   2034                     "|$dst {${mask}} {z}, $src1, $src2}"),
   2035                 [], itins.rr>, EVEX_4V, EVEX_KZ;
   2036   }
   2037   let mayLoad = 1 in {
   2038     def rm : AVX512BI<opc, MRMSrcMem, (outs RC:$dst),
   2039               (ins RC:$src1, x86memop:$src2),
   2040               !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   2041               []>, EVEX_4V;
   2042     def rmk : AVX512BI<opc, MRMSrcMem, (outs RC:$dst),
   2043                (ins KRC:$mask, RC:$src1, x86memop:$src2),
   2044                !strconcat(OpcodeStr,
   2045                    " \t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"),
   2046                [], itins.rm>, EVEX_4V, EVEX_K;
   2047     def rmkz : AVX512BI<opc, MRMSrcMem, (outs RC:$dst),
   2048                 (ins KRC:$mask, RC:$src1, x86memop:$src2),
   2049                 !strconcat(OpcodeStr,
   2050                     " \t{$src2, $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, $src2}"),
   2051                 [], itins.rm>, EVEX_4V, EVEX_KZ;
   2052     def rmb : AVX512BI<opc, MRMSrcMem, (outs RC:$dst),
   2053                (ins RC:$src1, x86scalar_mop:$src2),
   2054                !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr,
   2055                           ", $src1, $dst|$dst, $src1, ${src2}", BrdcstStr, "}"),
   2056                [], itins.rm>, EVEX_4V, EVEX_B;
   2057     def rmbk : AVX512BI<opc, MRMSrcMem, (outs RC:$dst),
   2058                 (ins KRC:$mask, RC:$src1, x86scalar_mop:$src2),
   2059                 !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr,
   2060                            ", $src1, $dst {${mask}}|$dst {${mask}}, $src1, ${src2}",
   2061                            BrdcstStr, "}"),
   2062                 [], itins.rm>, EVEX_4V, EVEX_B, EVEX_K;
   2063     def rmbkz : AVX512BI<opc, MRMSrcMem, (outs RC:$dst),
   2064                  (ins KRC:$mask, RC:$src1, x86scalar_mop:$src2),
   2065                  !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr,
   2066                             ", $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, ${src2}",
   2067                             BrdcstStr, "}"),
   2068                  [], itins.rm>, EVEX_4V, EVEX_B, EVEX_KZ;
   2069   }
   2070 }
   2071 
   2072 defm VPADDDZ : avx512_binop_rm<0xFE, "vpaddd", add, v16i32, VK16WM, VR512,
   2073                    memopv16i32, i512mem, loadi32, i32mem, "{1to16}",
   2074                    SSE_INTALU_ITINS_P, 1>, EVEX_V512, EVEX_CD8<32, CD8VF>;
   2075 
   2076 defm VPSUBDZ : avx512_binop_rm<0xFA, "vpsubd", sub, v16i32, VK16WM, VR512,
   2077                    memopv16i32, i512mem, loadi32, i32mem, "{1to16}",
   2078                    SSE_INTALU_ITINS_P, 0>, EVEX_V512, EVEX_CD8<32, CD8VF>;
   2079 
   2080 defm VPMULLDZ : avx512_binop_rm<0x40, "vpmulld", mul, v16i32, VK16WM, VR512,
   2081                    memopv16i32, i512mem, loadi32, i32mem, "{1to16}",
   2082                    SSE_INTALU_ITINS_P, 1>, T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>;
   2083 
   2084 defm VPADDQZ : avx512_binop_rm<0xD4, "vpaddq", add, v8i64, VK8WM, VR512,
   2085                    memopv8i64, i512mem, loadi64, i64mem, "{1to8}",
   2086                    SSE_INTALU_ITINS_P, 1>, EVEX_CD8<64, CD8VF>, EVEX_V512, VEX_W;
   2087 
   2088 defm VPSUBQZ : avx512_binop_rm<0xFB, "vpsubq", sub, v8i64, VK8WM, VR512,
   2089                    memopv8i64, i512mem, loadi64, i64mem, "{1to8}",
   2090                    SSE_INTALU_ITINS_P, 0>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
   2091 
   2092 defm VPMULDQZ : avx512_binop_rm2<0x28, "vpmuldq", v8i64, v16i32, VK8WM, VR512,
   2093                    memopv8i64, i512mem, loadi64, i64mem, "{1to8}",
   2094                    SSE_INTALU_ITINS_P, 1>, T8PD, EVEX_V512,
   2095                    EVEX_CD8<64, CD8VF>, VEX_W;
   2096 
   2097 defm VPMULUDQZ : avx512_binop_rm2<0xF4, "vpmuludq", v8i64, v16i32, VK8WM, VR512,
   2098                    memopv8i64, i512mem, loadi64, i64mem, "{1to8}",
   2099                    SSE_INTMUL_ITINS_P, 1>, EVEX_V512, EVEX_CD8<64, CD8VF>, VEX_W;
   2100 
   2101 def : Pat<(v8i64 (X86pmuludq (v16i32 VR512:$src1), (v16i32 VR512:$src2))),
   2102           (VPMULUDQZrr VR512:$src1, VR512:$src2)>;
   2103 
   2104 def : Pat<(v8i64 (int_x86_avx512_mask_pmulu_dq_512 (v16i32 VR512:$src1),
   2105            (v16i32 VR512:$src2), (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))),
   2106           (VPMULUDQZrr VR512:$src1, VR512:$src2)>;
   2107 def : Pat<(v8i64 (int_x86_avx512_mask_pmul_dq_512 (v16i32 VR512:$src1),
   2108            (v16i32 VR512:$src2), (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))),
   2109           (VPMULDQZrr VR512:$src1, VR512:$src2)>;
   2110 
   2111 defm VPMAXUDZ : avx512_binop_rm<0x3F, "vpmaxud", X86umax, v16i32, VK16WM, VR512,
   2112                    memopv16i32, i512mem, loadi32, i32mem, "{1to16}",
   2113                    SSE_INTALU_ITINS_P, 1>,
   2114                    T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>;
   2115 defm VPMAXUQZ : avx512_binop_rm<0x3F, "vpmaxuq", X86umax, v8i64, VK8WM, VR512,
   2116                    memopv8i64, i512mem, loadi64, i64mem, "{1to8}",
   2117                    SSE_INTALU_ITINS_P, 0>,
   2118                    T8PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
   2119 
   2120 defm VPMAXSDZ : avx512_binop_rm<0x3D, "vpmaxsd", X86smax, v16i32, VK16WM, VR512,
   2121                    memopv16i32, i512mem, loadi32, i32mem, "{1to16}",
   2122                    SSE_INTALU_ITINS_P, 1>,
   2123                    T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>;
   2124 defm VPMAXSQZ : avx512_binop_rm<0x3D, "vpmaxsq", X86smax, v8i64, VK8WM, VR512,
   2125                    memopv8i64, i512mem, loadi64, i64mem, "{1to8}",
   2126                    SSE_INTALU_ITINS_P, 0>,
   2127                    T8PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
   2128 
   2129 defm VPMINUDZ : avx512_binop_rm<0x3B, "vpminud", X86umin, v16i32, VK16WM, VR512,
   2130                    memopv16i32, i512mem, loadi32, i32mem, "{1to16}",
   2131                    SSE_INTALU_ITINS_P, 1>,
   2132                    T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>;
   2133 defm VPMINUQZ : avx512_binop_rm<0x3B, "vpminuq", X86umin, v8i64, VK8WM, VR512,
   2134                    memopv8i64, i512mem, loadi64, i64mem, "{1to8}",
   2135                    SSE_INTALU_ITINS_P, 0>,
   2136                    T8PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
   2137 
   2138 defm VPMINSDZ : avx512_binop_rm<0x39, "vpminsd", X86smin, v16i32, VK16WM, VR512,
   2139                    memopv16i32, i512mem, loadi32, i32mem, "{1to16}",
   2140                    SSE_INTALU_ITINS_P, 1>,
   2141                    T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>;
   2142 defm VPMINSQZ : avx512_binop_rm<0x39, "vpminsq", X86smin, v8i64, VK8WM, VR512,
   2143                    memopv8i64, i512mem, loadi64, i64mem, "{1to8}",
   2144                    SSE_INTALU_ITINS_P, 0>,
   2145                    T8PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
   2146 
   2147 def : Pat <(v16i32 (int_x86_avx512_mask_pmaxs_d_512 (v16i32 VR512:$src1),
   2148                     (v16i32 VR512:$src2), (v16i32 immAllZerosV), (i16 -1))),
   2149            (VPMAXSDZrr VR512:$src1, VR512:$src2)>;
   2150 def : Pat <(v16i32 (int_x86_avx512_mask_pmaxu_d_512 (v16i32 VR512:$src1),
   2151                     (v16i32 VR512:$src2), (v16i32 immAllZerosV), (i16 -1))),
   2152            (VPMAXUDZrr VR512:$src1, VR512:$src2)>;
   2153 def : Pat <(v8i64 (int_x86_avx512_mask_pmaxs_q_512 (v8i64 VR512:$src1),
   2154                 (v8i64 VR512:$src2), (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))),
   2155            (VPMAXSQZrr VR512:$src1, VR512:$src2)>;
   2156 def : Pat <(v8i64 (int_x86_avx512_mask_pmaxu_q_512 (v8i64 VR512:$src1),
   2157                 (v8i64 VR512:$src2), (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))),
   2158            (VPMAXUQZrr VR512:$src1, VR512:$src2)>;
   2159 def : Pat <(v16i32 (int_x86_avx512_mask_pmins_d_512 (v16i32 VR512:$src1),
   2160                     (v16i32 VR512:$src2), (v16i32 immAllZerosV), (i16 -1))),
   2161            (VPMINSDZrr VR512:$src1, VR512:$src2)>;
   2162 def : Pat <(v16i32 (int_x86_avx512_mask_pminu_d_512 (v16i32 VR512:$src1),
   2163                     (v16i32 VR512:$src2), (v16i32 immAllZerosV), (i16 -1))),
   2164            (VPMINUDZrr VR512:$src1, VR512:$src2)>;
   2165 def : Pat <(v8i64 (int_x86_avx512_mask_pmins_q_512 (v8i64 VR512:$src1),
   2166                 (v8i64 VR512:$src2), (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))),
   2167            (VPMINSQZrr VR512:$src1, VR512:$src2)>;
   2168 def : Pat <(v8i64 (int_x86_avx512_mask_pminu_q_512 (v8i64 VR512:$src1),
   2169                 (v8i64 VR512:$src2), (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))),
   2170            (VPMINUQZrr VR512:$src1, VR512:$src2)>;
   2171 //===----------------------------------------------------------------------===//
   2172 // AVX-512 - Unpack Instructions
   2173 //===----------------------------------------------------------------------===//
   2174 
   2175 multiclass avx512_unpack_fp<bits<8> opc, SDNode OpNode, ValueType vt,
   2176                                    PatFrag mem_frag, RegisterClass RC,
   2177                                    X86MemOperand x86memop, string asm,
   2178                                    Domain d> {
   2179     def rr : AVX512PI<opc, MRMSrcReg,
   2180                 (outs RC:$dst), (ins RC:$src1, RC:$src2),
   2181                 asm, [(set RC:$dst,
   2182                            (vt (OpNode RC:$src1, RC:$src2)))],
   2183                            d>, EVEX_4V;
   2184     def rm : AVX512PI<opc, MRMSrcMem,
   2185                 (outs RC:$dst), (ins RC:$src1, x86memop:$src2),
   2186                 asm, [(set RC:$dst,
   2187                        (vt (OpNode RC:$src1,
   2188                             (bitconvert (mem_frag addr:$src2)))))],
   2189                         d>, EVEX_4V;
   2190 }
   2191 
   2192 defm VUNPCKHPSZ: avx512_unpack_fp<0x15, X86Unpckh, v16f32, memopv8f64,
   2193       VR512, f512mem, "vunpckhps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
   2194       SSEPackedSingle>, PS, EVEX_V512, EVEX_CD8<32, CD8VF>;
   2195 defm VUNPCKHPDZ: avx512_unpack_fp<0x15, X86Unpckh, v8f64, memopv8f64,
   2196       VR512, f512mem, "vunpckhpd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
   2197       SSEPackedDouble>, PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
   2198 defm VUNPCKLPSZ: avx512_unpack_fp<0x14, X86Unpckl, v16f32, memopv8f64,
   2199       VR512, f512mem, "vunpcklps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
   2200       SSEPackedSingle>, PS, EVEX_V512, EVEX_CD8<32, CD8VF>;
   2201 defm VUNPCKLPDZ: avx512_unpack_fp<0x14, X86Unpckl, v8f64, memopv8f64,
   2202       VR512, f512mem, "vunpcklpd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
   2203       SSEPackedDouble>, PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
   2204 
   2205 multiclass avx512_unpack_int<bits<8> opc, string OpcodeStr, SDNode OpNode,
   2206                         ValueType OpVT, RegisterClass RC, PatFrag memop_frag,
   2207                         X86MemOperand x86memop> {
   2208   def rr : AVX512BI<opc, MRMSrcReg, (outs RC:$dst),
   2209        (ins RC:$src1, RC:$src2),
   2210        !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   2211        [(set RC:$dst, (OpVT (OpNode (OpVT RC:$src1), (OpVT RC:$src2))))], 
   2212        IIC_SSE_UNPCK>, EVEX_4V;
   2213   def rm : AVX512BI<opc, MRMSrcMem, (outs RC:$dst),
   2214        (ins RC:$src1, x86memop:$src2),
   2215        !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   2216        [(set RC:$dst, (OpVT (OpNode (OpVT RC:$src1),
   2217                                      (bitconvert (memop_frag addr:$src2)))))],
   2218                                      IIC_SSE_UNPCK>, EVEX_4V;
   2219 }
   2220 defm VPUNPCKLDQZ  : avx512_unpack_int<0x62, "vpunpckldq", X86Unpckl, v16i32,
   2221                                 VR512, memopv16i32, i512mem>, EVEX_V512,
   2222                                 EVEX_CD8<32, CD8VF>;
   2223 defm VPUNPCKLQDQZ : avx512_unpack_int<0x6C, "vpunpcklqdq", X86Unpckl, v8i64,
   2224                                 VR512, memopv8i64, i512mem>, EVEX_V512,
   2225                                 VEX_W, EVEX_CD8<64, CD8VF>;
   2226 defm VPUNPCKHDQZ  : avx512_unpack_int<0x6A, "vpunpckhdq", X86Unpckh, v16i32,
   2227                                 VR512, memopv16i32, i512mem>, EVEX_V512,
   2228                                 EVEX_CD8<32, CD8VF>;
   2229 defm VPUNPCKHQDQZ : avx512_unpack_int<0x6D, "vpunpckhqdq", X86Unpckh, v8i64,
   2230                                 VR512, memopv8i64, i512mem>, EVEX_V512,
   2231                                 VEX_W, EVEX_CD8<64, CD8VF>;
   2232 //===----------------------------------------------------------------------===//
   2233 // AVX-512 - PSHUFD
   2234 //
   2235 
   2236 multiclass avx512_pshuf_imm<bits<8> opc, string OpcodeStr, RegisterClass RC,
   2237                          SDNode OpNode, PatFrag mem_frag, 
   2238                          X86MemOperand x86memop, ValueType OpVT> {
   2239   def ri : AVX512Ii8<opc, MRMSrcReg, (outs RC:$dst),
   2240                      (ins RC:$src1, i8imm:$src2),
   2241                      !strconcat(OpcodeStr,
   2242                          " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   2243                      [(set RC:$dst,
   2244                        (OpVT (OpNode RC:$src1, (i8 imm:$src2))))]>,
   2245                      EVEX;
   2246   def mi : AVX512Ii8<opc, MRMSrcMem, (outs RC:$dst),
   2247                      (ins x86memop:$src1, i8imm:$src2),
   2248                      !strconcat(OpcodeStr,
   2249                          " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   2250                      [(set RC:$dst,
   2251                        (OpVT (OpNode (mem_frag addr:$src1),
   2252                               (i8 imm:$src2))))]>, EVEX;
   2253 }
   2254 
   2255 defm VPSHUFDZ : avx512_pshuf_imm<0x70, "vpshufd", VR512, X86PShufd, memopv16i32,
   2256                       i512mem, v16i32>, PD, EVEX_V512, EVEX_CD8<32, CD8VF>;
   2257 
   2258 let ExeDomain = SSEPackedSingle in
   2259 defm VPERMILPSZ : avx512_pshuf_imm<0x04, "vpermilps", VR512, X86VPermilp,
   2260                       memopv16f32, i512mem, v16f32>, TAPD, EVEX_V512,
   2261                       EVEX_CD8<32, CD8VF>;
   2262 let ExeDomain = SSEPackedDouble in
   2263 defm VPERMILPDZ : avx512_pshuf_imm<0x05, "vpermilpd", VR512, X86VPermilp,
   2264                       memopv8f64, i512mem, v8f64>, TAPD, EVEX_V512,
   2265                       VEX_W, EVEX_CD8<32, CD8VF>;
   2266 
   2267 def : Pat<(v16i32 (X86VPermilp VR512:$src1, (i8 imm:$imm))),
   2268           (VPERMILPSZri VR512:$src1, imm:$imm)>;
   2269 def : Pat<(v8i64 (X86VPermilp VR512:$src1, (i8 imm:$imm))),
   2270           (VPERMILPDZri VR512:$src1, imm:$imm)>;
   2271 
   2272 //===----------------------------------------------------------------------===//
   2273 // AVX-512  Logical Instructions
   2274 //===----------------------------------------------------------------------===//
   2275 
   2276 defm VPANDDZ : avx512_binop_rm<0xDB, "vpandd", and, v16i32, VK16WM, VR512, memopv16i32,
   2277                       i512mem, loadi32, i32mem, "{1to16}", SSE_BIT_ITINS_P, 1>,
   2278                       EVEX_V512, EVEX_CD8<32, CD8VF>;
   2279 defm VPANDQZ : avx512_binop_rm<0xDB, "vpandq", and, v8i64, VK8WM, VR512, memopv8i64,
   2280                       i512mem, loadi64, i64mem, "{1to8}", SSE_BIT_ITINS_P, 1>,
   2281                       EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
   2282 defm VPORDZ  : avx512_binop_rm<0xEB, "vpord", or, v16i32, VK16WM, VR512, memopv16i32,
   2283                       i512mem, loadi32, i32mem, "{1to16}", SSE_BIT_ITINS_P, 1>,
   2284                       EVEX_V512, EVEX_CD8<32, CD8VF>;
   2285 defm VPORQZ  : avx512_binop_rm<0xEB, "vporq", or, v8i64, VK8WM, VR512, memopv8i64,
   2286                       i512mem, loadi64, i64mem, "{1to8}", SSE_BIT_ITINS_P, 1>,
   2287                       EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
   2288 defm VPXORDZ : avx512_binop_rm<0xEF, "vpxord", xor, v16i32, VK16WM, VR512, memopv16i32,
   2289                       i512mem, loadi32, i32mem, "{1to16}", SSE_BIT_ITINS_P, 1>,
   2290                       EVEX_V512, EVEX_CD8<32, CD8VF>;
   2291 defm VPXORQZ : avx512_binop_rm<0xEF, "vpxorq", xor, v8i64, VK8WM, VR512, memopv8i64,
   2292                       i512mem, loadi64, i64mem, "{1to8}", SSE_BIT_ITINS_P, 1>,
   2293                       EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
   2294 defm VPANDNDZ : avx512_binop_rm<0xDF, "vpandnd", X86andnp, v16i32, VK16WM, VR512,
   2295                       memopv16i32, i512mem, loadi32, i32mem, "{1to16}",
   2296                       SSE_BIT_ITINS_P, 0>, EVEX_V512, EVEX_CD8<32, CD8VF>;
   2297 defm VPANDNQZ : avx512_binop_rm<0xDF, "vpandnq", X86andnp, v8i64, VK8WM, VR512,
   2298                       memopv8i64, i512mem, loadi64, i64mem, "{1to8}",
   2299                       SSE_BIT_ITINS_P, 0>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
   2300 
   2301 //===----------------------------------------------------------------------===//
   2302 // AVX-512  FP arithmetic
   2303 //===----------------------------------------------------------------------===//
   2304 
   2305 multiclass avx512_binop_s<bits<8> opc, string OpcodeStr, SDNode OpNode,
   2306                                   SizeItins itins> {
   2307   defm SSZ : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "ss"), OpNode, FR32X,
   2308                              f32mem, itins.s, 0>, XS, EVEX_4V, VEX_LIG,
   2309                              EVEX_CD8<32, CD8VT1>;
   2310   defm SDZ : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "sd"), OpNode, FR64X,
   2311                              f64mem, itins.d, 0>, XD, VEX_W, EVEX_4V, VEX_LIG,
   2312                              EVEX_CD8<64, CD8VT1>;
   2313 }
   2314 
   2315 let isCommutable = 1 in {
   2316 defm VADD : avx512_binop_s<0x58, "add", fadd, SSE_ALU_ITINS_S>;
   2317 defm VMUL : avx512_binop_s<0x59, "mul", fmul, SSE_ALU_ITINS_S>;
   2318 defm VMIN : avx512_binop_s<0x5D, "min", X86fmin, SSE_ALU_ITINS_S>;
   2319 defm VMAX : avx512_binop_s<0x5F, "max", X86fmax, SSE_ALU_ITINS_S>;
   2320 }
   2321 let isCommutable = 0 in {
   2322 defm VSUB : avx512_binop_s<0x5C, "sub", fsub, SSE_ALU_ITINS_S>;
   2323 defm VDIV : avx512_binop_s<0x5E, "div", fdiv, SSE_ALU_ITINS_S>;
   2324 }
   2325 
   2326 multiclass avx512_fp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
   2327                            RegisterClass KRC,
   2328                            RegisterClass RC, ValueType vt,
   2329                            X86MemOperand x86memop, PatFrag mem_frag,
   2330                            X86MemOperand x86scalar_mop, PatFrag scalar_mfrag,
   2331                            string BrdcstStr,
   2332                            Domain d, OpndItins itins, bit commutable> {
   2333   let isCommutable = commutable in {
   2334     def rr : PI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2),
   2335        !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   2336        [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))], itins.rr, d>,
   2337        EVEX_4V;
   2338 
   2339     def rrk: PI<opc, MRMSrcReg, (outs RC:$dst), (ins KRC:$mask, RC:$src1, RC:$src2),
   2340        !strconcat(OpcodeStr,
   2341            " \t{$src2, $src1, $dst {${mask}} |$dst {${mask}}, $src1, $src2}"),
   2342        [], itins.rr, d>, EVEX_4V, EVEX_K;
   2343 
   2344     def rrkz: PI<opc, MRMSrcReg, (outs RC:$dst), (ins KRC:$mask, RC:$src1, RC:$src2),
   2345        !strconcat(OpcodeStr,
   2346            " \t{$src2, $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, $src2}"),
   2347        [], itins.rr, d>, EVEX_4V, EVEX_KZ;
   2348   }
   2349 
   2350   let mayLoad = 1 in {
   2351     def rm : PI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2),
   2352        !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   2353        [(set RC:$dst, (OpNode RC:$src1, (mem_frag addr:$src2)))],
   2354           itins.rm, d>, EVEX_4V;
   2355 
   2356     def rmb : PI<opc, MRMSrcMem, (outs RC:$dst),
   2357        (ins RC:$src1, x86scalar_mop:$src2),
   2358        !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr,
   2359            ", $src1, $dst|$dst, $src1, ${src2}", BrdcstStr, "}"),
   2360        [(set RC:$dst, (OpNode RC:$src1, 
   2361                        (vt (X86VBroadcast (scalar_mfrag addr:$src2)))))],
   2362        itins.rm, d>, EVEX_4V, EVEX_B;
   2363 
   2364     def rmk : PI<opc, MRMSrcMem, (outs RC:$dst),
   2365        (ins KRC:$mask, RC:$src1, x86memop:$src2), !strconcat(OpcodeStr,
   2366            "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"),
   2367        [], itins.rm, d>, EVEX_4V, EVEX_K;
   2368 
   2369     def rmkz : PI<opc, MRMSrcMem, (outs RC:$dst),
   2370        (ins KRC:$mask, RC:$src1, x86memop:$src2), !strconcat(OpcodeStr,
   2371            "\t{$src2, $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, $src2}"),
   2372        [], itins.rm, d>, EVEX_4V, EVEX_KZ;
   2373 
   2374     def rmbk : PI<opc, MRMSrcMem, (outs RC:$dst),
   2375        (ins KRC:$mask, RC:$src1, x86scalar_mop:$src2), !strconcat(OpcodeStr,
   2376            " \t{${src2}", BrdcstStr,
   2377            ", $src1, $dst {${mask}}|$dst {${mask}}, $src1, ${src2}", BrdcstStr, "}"),
   2378        [], itins.rm, d>, EVEX_4V, EVEX_B, EVEX_K;
   2379 
   2380     def rmbkz : PI<opc, MRMSrcMem, (outs RC:$dst),
   2381        (ins KRC:$mask, RC:$src1, x86scalar_mop:$src2), !strconcat(OpcodeStr,
   2382            " \t{${src2}", BrdcstStr,
   2383            ", $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, ${src2}",
   2384            BrdcstStr, "}"),
   2385        [], itins.rm, d>, EVEX_4V, EVEX_B, EVEX_KZ;
   2386   }
   2387 }
   2388 
   2389 defm VADDPSZ : avx512_fp_packed<0x58, "addps", fadd, VK16WM, VR512, v16f32, f512mem,
   2390                    memopv16f32, f32mem, loadf32, "{1to16}", SSEPackedSingle, 
   2391                    SSE_ALU_ITINS_P.s, 1>, EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
   2392                    
   2393 defm VADDPDZ : avx512_fp_packed<0x58, "addpd", fadd, VK8WM, VR512, v8f64, f512mem,
   2394                    memopv8f64, f64mem, loadf64, "{1to8}", SSEPackedDouble,
   2395                    SSE_ALU_ITINS_P.d, 1>,
   2396                    EVEX_V512, PD, VEX_W, EVEX_CD8<64, CD8VF>;
   2397 
   2398 defm VMULPSZ : avx512_fp_packed<0x59, "mulps", fmul, VK16WM, VR512, v16f32, f512mem,
   2399                    memopv16f32, f32mem, loadf32, "{1to16}", SSEPackedSingle,
   2400                    SSE_ALU_ITINS_P.s, 1>, EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
   2401 defm VMULPDZ : avx512_fp_packed<0x59, "mulpd", fmul, VK8WM, VR512, v8f64, f512mem,
   2402                    memopv8f64, f64mem, loadf64, "{1to8}", SSEPackedDouble,
   2403                    SSE_ALU_ITINS_P.d, 1>,
   2404                    EVEX_V512, PD, VEX_W, EVEX_CD8<64, CD8VF>;
   2405 
   2406 defm VMINPSZ : avx512_fp_packed<0x5D, "minps", X86fmin, VK16WM, VR512, v16f32, f512mem,
   2407                    memopv16f32, f32mem, loadf32, "{1to16}", SSEPackedSingle,
   2408                    SSE_ALU_ITINS_P.s, 1>,
   2409                    EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
   2410 defm VMAXPSZ : avx512_fp_packed<0x5F, "maxps", X86fmax, VK16WM, VR512, v16f32, f512mem,
   2411                    memopv16f32, f32mem, loadf32, "{1to16}", SSEPackedSingle,
   2412                    SSE_ALU_ITINS_P.s, 1>,
   2413                    EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
   2414 
   2415 defm VMINPDZ : avx512_fp_packed<0x5D, "minpd", X86fmin, VK8WM, VR512, v8f64, f512mem,
   2416                    memopv8f64, f64mem, loadf64, "{1to8}", SSEPackedDouble,
   2417                    SSE_ALU_ITINS_P.d, 1>,
   2418                    EVEX_V512, PD, VEX_W, EVEX_CD8<64, CD8VF>;
   2419 defm VMAXPDZ : avx512_fp_packed<0x5F, "maxpd", X86fmax, VK8WM, VR512, v8f64, f512mem,
   2420                    memopv8f64, f64mem, loadf64, "{1to8}", SSEPackedDouble,
   2421                    SSE_ALU_ITINS_P.d, 1>,
   2422                    EVEX_V512, PD, VEX_W, EVEX_CD8<64, CD8VF>;
   2423 
   2424 defm VSUBPSZ : avx512_fp_packed<0x5C, "subps", fsub, VK16WM, VR512, v16f32, f512mem,
   2425                    memopv16f32, f32mem, loadf32, "{1to16}", SSEPackedSingle,
   2426                    SSE_ALU_ITINS_P.s, 0>, EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
   2427 defm VDIVPSZ : avx512_fp_packed<0x5E, "divps", fdiv, VK16WM, VR512, v16f32, f512mem,
   2428                    memopv16f32, f32mem, loadf32, "{1to16}", SSEPackedSingle,
   2429                    SSE_ALU_ITINS_P.s, 0>, EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
   2430 
   2431 defm VSUBPDZ : avx512_fp_packed<0x5C, "subpd", fsub, VK8WM, VR512, v8f64, f512mem,
   2432                    memopv8f64, f64mem, loadf64, "{1to8}", SSEPackedDouble,
   2433                    SSE_ALU_ITINS_P.d, 0>, 
   2434                    EVEX_V512, PD, VEX_W, EVEX_CD8<64, CD8VF>;
   2435 defm VDIVPDZ : avx512_fp_packed<0x5E, "divpd", fdiv, VK8WM, VR512, v8f64, f512mem,
   2436                    memopv8f64, f64mem, loadf64, "{1to8}", SSEPackedDouble,
   2437                    SSE_ALU_ITINS_P.d, 0>, 
   2438                    EVEX_V512, PD, VEX_W, EVEX_CD8<64, CD8VF>;
   2439 
   2440 def : Pat<(v16f32 (int_x86_avx512_mask_max_ps_512 (v16f32 VR512:$src1),
   2441                    (v16f32 VR512:$src2), (bc_v16f32 (v16i32 immAllZerosV)),
   2442                    (i16 -1), FROUND_CURRENT)),
   2443           (VMAXPSZrr VR512:$src1, VR512:$src2)>;
   2444 
   2445 def : Pat<(v8f64 (int_x86_avx512_mask_max_pd_512 (v8f64 VR512:$src1),
   2446                    (v8f64 VR512:$src2), (bc_v8f64 (v16i32 immAllZerosV)),
   2447                    (i8 -1), FROUND_CURRENT)),
   2448           (VMAXPDZrr VR512:$src1, VR512:$src2)>;
   2449 
   2450 def : Pat<(v16f32 (int_x86_avx512_mask_min_ps_512 (v16f32 VR512:$src1),
   2451                    (v16f32 VR512:$src2), (bc_v16f32 (v16i32 immAllZerosV)),
   2452                    (i16 -1), FROUND_CURRENT)),
   2453           (VMINPSZrr VR512:$src1, VR512:$src2)>;
   2454 
   2455 def : Pat<(v8f64 (int_x86_avx512_mask_min_pd_512 (v8f64 VR512:$src1),
   2456                    (v8f64 VR512:$src2), (bc_v8f64 (v16i32 immAllZerosV)),
   2457                    (i8 -1), FROUND_CURRENT)),
   2458           (VMINPDZrr VR512:$src1, VR512:$src2)>;
   2459 //===----------------------------------------------------------------------===//
   2460 // AVX-512  VPTESTM instructions
   2461 //===----------------------------------------------------------------------===//
   2462 
   2463 multiclass avx512_vptest<bits<8> opc, string OpcodeStr, RegisterClass KRC, 
   2464               RegisterClass RC, X86MemOperand x86memop, PatFrag memop_frag, 
   2465               SDNode OpNode, ValueType vt> {
   2466   def rr : AVX512PI<opc, MRMSrcReg,
   2467              (outs KRC:$dst), (ins RC:$src1, RC:$src2), 
   2468              !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   2469              [(set KRC:$dst, (OpNode (vt RC:$src1), (vt RC:$src2)))],
   2470              SSEPackedInt>, EVEX_4V;
   2471   def rm : AVX512PI<opc, MRMSrcMem,
   2472              (outs KRC:$dst), (ins RC:$src1, x86memop:$src2), 
   2473              !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   2474              [(set KRC:$dst, (OpNode (vt RC:$src1), 
   2475               (bitconvert (memop_frag addr:$src2))))], SSEPackedInt>, EVEX_4V;
   2476 }
   2477 
   2478 defm VPTESTMDZ  : avx512_vptest<0x27, "vptestmd", VK16, VR512,  f512mem,
   2479                               memopv16i32, X86testm, v16i32>, T8PD, EVEX_V512,
   2480                               EVEX_CD8<32, CD8VF>;
   2481 defm VPTESTMQZ  : avx512_vptest<0x27, "vptestmq", VK8, VR512,  f512mem,
   2482                               memopv8i64, X86testm, v8i64>, T8PD, EVEX_V512, VEX_W,
   2483                               EVEX_CD8<64, CD8VF>;
   2484 
   2485 let Predicates = [HasCDI] in {
   2486 defm VPTESTNMDZ  : avx512_vptest<0x27, "vptestnmd", VK16, VR512,  f512mem,
   2487                               memopv16i32, X86testnm, v16i32>, T8XS, EVEX_V512,
   2488                               EVEX_CD8<32, CD8VF>;
   2489 defm VPTESTNMQZ  : avx512_vptest<0x27, "vptestnmq", VK8, VR512,  f512mem,
   2490                               memopv8i64, X86testnm, v8i64>, T8XS, EVEX_V512, VEX_W,
   2491                               EVEX_CD8<64, CD8VF>;
   2492 }
   2493 
   2494 def : Pat <(i16 (int_x86_avx512_mask_ptestm_d_512 (v16i32 VR512:$src1),
   2495                  (v16i32 VR512:$src2), (i16 -1))),
   2496                  (COPY_TO_REGCLASS (VPTESTMDZrr VR512:$src1, VR512:$src2), GR16)>;
   2497 
   2498 def : Pat <(i8 (int_x86_avx512_mask_ptestm_q_512 (v8i64 VR512:$src1),
   2499                  (v8i64 VR512:$src2), (i8 -1))),
   2500                  (COPY_TO_REGCLASS (VPTESTMQZrr VR512:$src1, VR512:$src2), GR8)>;
   2501 //===----------------------------------------------------------------------===//
   2502 // AVX-512  Shift instructions
   2503 //===----------------------------------------------------------------------===//
   2504 multiclass avx512_shift_rmi<bits<8> opc, Format ImmFormR, Format ImmFormM,
   2505                          string OpcodeStr, SDNode OpNode, RegisterClass RC,
   2506                          ValueType vt, X86MemOperand x86memop, PatFrag mem_frag,
   2507                          RegisterClass KRC> {
   2508   def ri : AVX512BIi8<opc, ImmFormR, (outs RC:$dst),
   2509        (ins RC:$src1, i8imm:$src2),
   2510            !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   2511        [(set RC:$dst, (vt (OpNode RC:$src1, (i8 imm:$src2))))],
   2512         SSE_INTSHIFT_ITINS_P.rr>, EVEX_4V;
   2513   def rik : AVX512BIi8<opc, ImmFormR, (outs RC:$dst),
   2514        (ins KRC:$mask, RC:$src1, i8imm:$src2),
   2515            !strconcat(OpcodeStr,
   2516                 " \t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"),
   2517        [], SSE_INTSHIFT_ITINS_P.rr>, EVEX_4V, EVEX_K;
   2518   def mi: AVX512BIi8<opc, ImmFormM, (outs RC:$dst),
   2519        (ins x86memop:$src1, i8imm:$src2),
   2520            !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   2521        [(set RC:$dst, (OpNode (mem_frag addr:$src1),
   2522                      (i8 imm:$src2)))], SSE_INTSHIFT_ITINS_P.rm>, EVEX_4V;
   2523   def mik: AVX512BIi8<opc, ImmFormM, (outs RC:$dst),
   2524        (ins KRC:$mask, x86memop:$src1, i8imm:$src2),
   2525            !strconcat(OpcodeStr,
   2526                 " \t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"),
   2527        [], SSE_INTSHIFT_ITINS_P.rm>, EVEX_4V, EVEX_K;
   2528 }
   2529 
   2530 multiclass avx512_shift_rrm<bits<8> opc, string OpcodeStr, SDNode OpNode,
   2531                           RegisterClass RC, ValueType vt, ValueType SrcVT,
   2532                           PatFrag bc_frag, RegisterClass KRC> {
   2533   // src2 is always 128-bit
   2534   def rr : AVX512BI<opc, MRMSrcReg, (outs RC:$dst),
   2535        (ins RC:$src1, VR128X:$src2),
   2536            !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   2537        [(set RC:$dst, (vt (OpNode RC:$src1, (SrcVT VR128X:$src2))))],
   2538         SSE_INTSHIFT_ITINS_P.rr>, EVEX_4V;
   2539   def rrk : AVX512BI<opc, MRMSrcReg, (outs RC:$dst),
   2540        (ins KRC:$mask, RC:$src1, VR128X:$src2),
   2541            !strconcat(OpcodeStr,
   2542                 " \t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"),
   2543        [], SSE_INTSHIFT_ITINS_P.rr>, EVEX_4V, EVEX_K;
   2544   def rm : AVX512BI<opc, MRMSrcMem, (outs RC:$dst),
   2545        (ins RC:$src1, i128mem:$src2),
   2546            !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   2547        [(set RC:$dst, (vt (OpNode RC:$src1,
   2548                        (bc_frag (memopv2i64 addr:$src2)))))],
   2549                         SSE_INTSHIFT_ITINS_P.rm>, EVEX_4V;
   2550   def rmk : AVX512BI<opc, MRMSrcMem, (outs RC:$dst),
   2551        (ins KRC:$mask, RC:$src1, i128mem:$src2),
   2552            !strconcat(OpcodeStr,
   2553                 " \t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}"),
   2554        [], SSE_INTSHIFT_ITINS_P.rm>, EVEX_4V, EVEX_K;
   2555 }
   2556 
   2557 defm VPSRLDZ : avx512_shift_rmi<0x72, MRM2r, MRM2m, "vpsrld", X86vsrli,
   2558                            VR512, v16i32, i512mem, memopv16i32, VK16WM>,
   2559                            EVEX_V512, EVEX_CD8<32, CD8VF>;
   2560 defm VPSRLDZ : avx512_shift_rrm<0xD2, "vpsrld", X86vsrl,
   2561                            VR512, v16i32, v4i32, bc_v4i32, VK16WM>, EVEX_V512,
   2562                            EVEX_CD8<32, CD8VQ>;
   2563                            
   2564 defm VPSRLQZ : avx512_shift_rmi<0x73, MRM2r, MRM2m, "vpsrlq", X86vsrli,
   2565                            VR512, v8i64, i512mem, memopv8i64, VK8WM>, EVEX_V512,
   2566                            EVEX_CD8<64, CD8VF>, VEX_W;
   2567 defm VPSRLQZ : avx512_shift_rrm<0xD3, "vpsrlq", X86vsrl,
   2568                            VR512, v8i64, v2i64, bc_v2i64, VK8WM>, EVEX_V512,
   2569                            EVEX_CD8<64, CD8VQ>, VEX_W;
   2570 
   2571 defm VPSLLDZ : avx512_shift_rmi<0x72, MRM6r, MRM6m, "vpslld", X86vshli,
   2572                            VR512, v16i32, i512mem, memopv16i32, VK16WM>, EVEX_V512,
   2573                            EVEX_CD8<32, CD8VF>;
   2574 defm VPSLLDZ : avx512_shift_rrm<0xF2, "vpslld", X86vshl,
   2575                            VR512, v16i32, v4i32, bc_v4i32, VK16WM>, EVEX_V512,
   2576                            EVEX_CD8<32, CD8VQ>;
   2577                            
   2578 defm VPSLLQZ : avx512_shift_rmi<0x73, MRM6r, MRM6m, "vpsllq", X86vshli,
   2579                            VR512, v8i64, i512mem, memopv8i64, VK8WM>, EVEX_V512,
   2580                            EVEX_CD8<64, CD8VF>, VEX_W;
   2581 defm VPSLLQZ : avx512_shift_rrm<0xF3, "vpsllq", X86vshl,
   2582                            VR512, v8i64, v2i64, bc_v2i64, VK8WM>, EVEX_V512,
   2583                            EVEX_CD8<64, CD8VQ>, VEX_W;
   2584 
   2585 defm VPSRADZ : avx512_shift_rmi<0x72, MRM4r, MRM4m, "vpsrad", X86vsrai,
   2586                            VR512, v16i32, i512mem, memopv16i32, VK16WM>,
   2587                            EVEX_V512, EVEX_CD8<32, CD8VF>;
   2588 defm VPSRADZ : avx512_shift_rrm<0xE2, "vpsrad", X86vsra,
   2589                            VR512, v16i32, v4i32, bc_v4i32, VK16WM>, EVEX_V512,
   2590                            EVEX_CD8<32, CD8VQ>;
   2591                            
   2592 defm VPSRAQZ : avx512_shift_rmi<0x72, MRM4r, MRM4m, "vpsraq", X86vsrai,
   2593                            VR512, v8i64, i512mem, memopv8i64, VK8WM>, EVEX_V512,
   2594                            EVEX_CD8<64, CD8VF>, VEX_W;
   2595 defm VPSRAQZ : avx512_shift_rrm<0xE2, "vpsraq", X86vsra,
   2596                            VR512, v8i64, v2i64, bc_v2i64, VK8WM>, EVEX_V512,
   2597                            EVEX_CD8<64, CD8VQ>, VEX_W;
   2598 
   2599 //===-------------------------------------------------------------------===//
   2600 // Variable Bit Shifts
   2601 //===-------------------------------------------------------------------===//
   2602 multiclass avx512_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode,
   2603                            RegisterClass RC, ValueType vt,
   2604                            X86MemOperand x86memop, PatFrag mem_frag> {
   2605   def rr  : AVX5128I<opc, MRMSrcReg, (outs RC:$dst),
   2606              (ins RC:$src1, RC:$src2),
   2607              !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   2608              [(set RC:$dst,
   2609                (vt (OpNode RC:$src1, (vt RC:$src2))))]>,
   2610              EVEX_4V;
   2611   def rm  : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
   2612              (ins RC:$src1, x86memop:$src2),
   2613              !strconcat(OpcodeStr, " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   2614              [(set RC:$dst,
   2615                (vt (OpNode RC:$src1, (mem_frag addr:$src2))))]>,
   2616              EVEX_4V;
   2617 }
   2618 
   2619 defm VPSLLVDZ : avx512_var_shift<0x47, "vpsllvd", shl, VR512, v16i32, 
   2620                                i512mem, memopv16i32>, EVEX_V512,
   2621                                EVEX_CD8<32, CD8VF>;
   2622 defm VPSLLVQZ : avx512_var_shift<0x47, "vpsllvq", shl, VR512, v8i64, 
   2623                                i512mem, memopv8i64>, EVEX_V512, VEX_W,
   2624                                EVEX_CD8<64, CD8VF>;
   2625 defm VPSRLVDZ : avx512_var_shift<0x45, "vpsrlvd", srl, VR512, v16i32, 
   2626                                i512mem, memopv16i32>, EVEX_V512,
   2627                                EVEX_CD8<32, CD8VF>;
   2628 defm VPSRLVQZ : avx512_var_shift<0x45, "vpsrlvq", srl, VR512, v8i64, 
   2629                                i512mem, memopv8i64>, EVEX_V512, VEX_W,
   2630                                EVEX_CD8<64, CD8VF>;
   2631 defm VPSRAVDZ : avx512_var_shift<0x46, "vpsravd", sra, VR512, v16i32, 
   2632                                i512mem, memopv16i32>, EVEX_V512,
   2633                                EVEX_CD8<32, CD8VF>;
   2634 defm VPSRAVQZ : avx512_var_shift<0x46, "vpsravq", sra, VR512, v8i64, 
   2635                                i512mem, memopv8i64>, EVEX_V512, VEX_W,
   2636                                EVEX_CD8<64, CD8VF>;
   2637 
   2638 //===----------------------------------------------------------------------===//
   2639 // AVX-512 - MOVDDUP
   2640 //===----------------------------------------------------------------------===//
   2641 
   2642 multiclass avx512_movddup<string OpcodeStr, RegisterClass RC, ValueType VT, 
   2643                         X86MemOperand x86memop, PatFrag memop_frag> {
   2644 def rr  : AVX512PDI<0x12, MRMSrcReg, (outs RC:$dst), (ins RC:$src),
   2645                     !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
   2646                     [(set RC:$dst, (VT (X86Movddup RC:$src)))]>, EVEX;
   2647 def rm  : AVX512PDI<0x12, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src),
   2648                     !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
   2649                     [(set RC:$dst,
   2650                       (VT (X86Movddup (memop_frag addr:$src))))]>, EVEX;
   2651 }
   2652 
   2653 defm VMOVDDUPZ : avx512_movddup<"vmovddup", VR512, v8f64, f512mem, memopv8f64>,
   2654                  VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
   2655 def : Pat<(X86Movddup (v8f64 (scalar_to_vector (loadf64 addr:$src)))),
   2656           (VMOVDDUPZrm addr:$src)>;
   2657 
   2658 //===---------------------------------------------------------------------===//
   2659 // Replicate Single FP - MOVSHDUP and MOVSLDUP
   2660 //===---------------------------------------------------------------------===//
   2661 multiclass avx512_replicate_sfp<bits<8> op, SDNode OpNode, string OpcodeStr,
   2662                               ValueType vt, RegisterClass RC, PatFrag mem_frag,
   2663                               X86MemOperand x86memop> {
   2664   def rr : AVX512XSI<op, MRMSrcReg, (outs RC:$dst), (ins RC:$src),
   2665                     !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
   2666                       [(set RC:$dst, (vt (OpNode RC:$src)))]>, EVEX;
   2667   let mayLoad = 1 in
   2668   def rm : AVX512XSI<op, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src),
   2669                     !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
   2670                       [(set RC:$dst, (OpNode (mem_frag addr:$src)))]>, EVEX;
   2671 }
   2672 
   2673 defm VMOVSHDUPZ  : avx512_replicate_sfp<0x16, X86Movshdup, "vmovshdup",
   2674                        v16f32, VR512, memopv16f32, f512mem>, EVEX_V512,
   2675                        EVEX_CD8<32, CD8VF>;
   2676 defm VMOVSLDUPZ  : avx512_replicate_sfp<0x12, X86Movsldup, "vmovsldup",
   2677                        v16f32, VR512, memopv16f32, f512mem>, EVEX_V512,
   2678                        EVEX_CD8<32, CD8VF>;
   2679 
   2680 def : Pat<(v16i32 (X86Movshdup VR512:$src)), (VMOVSHDUPZrr VR512:$src)>;
   2681 def : Pat<(v16i32 (X86Movshdup (memopv16i32 addr:$src))),
   2682            (VMOVSHDUPZrm addr:$src)>;
   2683 def : Pat<(v16i32 (X86Movsldup VR512:$src)), (VMOVSLDUPZrr VR512:$src)>;
   2684 def : Pat<(v16i32 (X86Movsldup (memopv16i32 addr:$src))),
   2685            (VMOVSLDUPZrm addr:$src)>;
   2686 
   2687 //===----------------------------------------------------------------------===//
   2688 // Move Low to High and High to Low packed FP Instructions
   2689 //===----------------------------------------------------------------------===//
   2690 def VMOVLHPSZrr : AVX512PSI<0x16, MRMSrcReg, (outs VR128X:$dst),
   2691           (ins VR128X:$src1, VR128X:$src2),
   2692           "vmovlhps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
   2693           [(set VR128X:$dst, (v4f32 (X86Movlhps VR128X:$src1, VR128X:$src2)))],
   2694            IIC_SSE_MOV_LH>, EVEX_4V;
   2695 def VMOVHLPSZrr : AVX512PSI<0x12, MRMSrcReg, (outs VR128X:$dst),
   2696           (ins VR128X:$src1, VR128X:$src2),
   2697           "vmovhlps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
   2698           [(set VR128X:$dst, (v4f32 (X86Movhlps VR128X:$src1, VR128X:$src2)))],
   2699           IIC_SSE_MOV_LH>, EVEX_4V;
   2700 
   2701 let Predicates = [HasAVX512] in {
   2702   // MOVLHPS patterns
   2703   def : Pat<(v4i32 (X86Movlhps VR128X:$src1, VR128X:$src2)),
   2704             (VMOVLHPSZrr VR128X:$src1, VR128X:$src2)>;
   2705   def : Pat<(v2i64 (X86Movlhps VR128X:$src1, VR128X:$src2)),
   2706             (VMOVLHPSZrr (v2i64 VR128X:$src1), VR128X:$src2)>;
   2707 
   2708   // MOVHLPS patterns
   2709   def : Pat<(v4i32 (X86Movhlps VR128X:$src1, VR128X:$src2)),
   2710             (VMOVHLPSZrr VR128X:$src1, VR128X:$src2)>;
   2711 }
   2712 
   2713 //===----------------------------------------------------------------------===//
   2714 // FMA - Fused Multiply Operations
   2715 //
   2716 let Constraints = "$src1 = $dst" in {
   2717 multiclass avx512_fma3p_rm<bits<8> opc, string OpcodeStr,
   2718             RegisterClass RC, X86MemOperand x86memop,
   2719             PatFrag mem_frag, X86MemOperand x86scalar_mop, PatFrag scalar_mfrag,
   2720             string BrdcstStr, SDNode OpNode, ValueType OpVT> {
   2721   def r: AVX512FMA3<opc, MRMSrcReg, (outs RC:$dst),
   2722           (ins RC:$src1, RC:$src2, RC:$src3),
   2723           !strconcat(OpcodeStr," \t{$src3, $src2, $dst|$dst, $src2, $src3}"),
   2724           [(set RC:$dst, (OpVT(OpNode RC:$src1, RC:$src2, RC:$src3)))]>;
   2725 
   2726   let mayLoad = 1 in
   2727   def m: AVX512FMA3<opc, MRMSrcMem, (outs RC:$dst),
   2728           (ins RC:$src1, RC:$src2, x86memop:$src3),
   2729           !strconcat(OpcodeStr, " \t{$src3, $src2, $dst|$dst, $src2, $src3}"),
   2730           [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2,
   2731                                                (mem_frag addr:$src3))))]>;
   2732    def mb: AVX512FMA3<opc, MRMSrcMem, (outs RC:$dst),
   2733            (ins RC:$src1, RC:$src2, x86scalar_mop:$src3),
   2734            !strconcat(OpcodeStr, " \t{${src3}", BrdcstStr,
   2735             ", $src2, $dst|$dst, $src2, ${src3}", BrdcstStr, "}"),
   2736            [(set RC:$dst, (OpNode RC:$src1, RC:$src2,
   2737            (OpVT (X86VBroadcast (scalar_mfrag addr:$src3)))))]>, EVEX_B;
   2738 }
   2739 } // Constraints = "$src1 = $dst"
   2740 
   2741 let ExeDomain = SSEPackedSingle in {
   2742   defm VFMADD213PSZ    : avx512_fma3p_rm<0xA8, "vfmadd213ps", VR512, f512mem,
   2743                                     memopv16f32, f32mem, loadf32, "{1to16}",
   2744                                     X86Fmadd, v16f32>, EVEX_V512,
   2745                                     EVEX_CD8<32, CD8VF>;
   2746   defm VFMSUB213PSZ    : avx512_fma3p_rm<0xAA, "vfmsub213ps", VR512, f512mem,
   2747                                     memopv16f32, f32mem, loadf32, "{1to16}",
   2748                                     X86Fmsub, v16f32>, EVEX_V512,
   2749                                     EVEX_CD8<32, CD8VF>;
   2750   defm VFMADDSUB213PSZ : avx512_fma3p_rm<0xA6, "vfmaddsub213ps", VR512, f512mem,
   2751                                     memopv16f32, f32mem, loadf32, "{1to16}",
   2752                                     X86Fmaddsub, v16f32>,
   2753                                     EVEX_V512, EVEX_CD8<32, CD8VF>;
   2754   defm VFMSUBADD213PSZ : avx512_fma3p_rm<0xA7, "vfmsubadd213ps", VR512, f512mem,
   2755                                     memopv16f32, f32mem, loadf32, "{1to16}",
   2756                                     X86Fmsubadd, v16f32>,
   2757                                     EVEX_V512, EVEX_CD8<32, CD8VF>;
   2758   defm VFNMADD213PSZ   : avx512_fma3p_rm<0xAC, "vfnmadd213ps", VR512, f512mem,
   2759                                     memopv16f32, f32mem, loadf32, "{1to16}",
   2760                                     X86Fnmadd, v16f32>, EVEX_V512,
   2761                                     EVEX_CD8<32, CD8VF>;
   2762   defm VFNMSUB213PSZ   : avx512_fma3p_rm<0xAE, "vfnmsub213ps", VR512, f512mem,
   2763                                     memopv16f32, f32mem, loadf32, "{1to16}",
   2764                                     X86Fnmsub, v16f32>, EVEX_V512,
   2765                                     EVEX_CD8<32, CD8VF>;
   2766 }
   2767 let ExeDomain = SSEPackedDouble in {
   2768   defm VFMADD213PDZ    : avx512_fma3p_rm<0xA8, "vfmadd213pd", VR512, f512mem,
   2769                                     memopv8f64, f64mem, loadf64, "{1to8}",
   2770                                     X86Fmadd, v8f64>, EVEX_V512,
   2771                                     VEX_W, EVEX_CD8<64, CD8VF>;
   2772   defm VFMSUB213PDZ    : avx512_fma3p_rm<0xAA, "vfmsub213pd", VR512, f512mem,
   2773                                     memopv8f64, f64mem, loadf64, "{1to8}",
   2774                                     X86Fmsub, v8f64>, EVEX_V512, VEX_W,
   2775                                     EVEX_CD8<64, CD8VF>;
   2776   defm VFMADDSUB213PDZ : avx512_fma3p_rm<0xA6, "vfmaddsub213pd", VR512, f512mem,
   2777                                     memopv8f64, f64mem, loadf64, "{1to8}",
   2778                                     X86Fmaddsub, v8f64>, EVEX_V512, VEX_W,
   2779                                     EVEX_CD8<64, CD8VF>;
   2780   defm VFMSUBADD213PDZ : avx512_fma3p_rm<0xA7, "vfmsubadd213pd", VR512, f512mem,
   2781                                     memopv8f64, f64mem, loadf64, "{1to8}",
   2782                                     X86Fmsubadd, v8f64>, EVEX_V512, VEX_W,
   2783                                     EVEX_CD8<64, CD8VF>;
   2784   defm VFNMADD213PDZ : avx512_fma3p_rm<0xAC, "vfnmadd213pd", VR512, f512mem,
   2785                                   memopv8f64, f64mem, loadf64, "{1to8}",
   2786                                   X86Fnmadd, v8f64>, EVEX_V512, VEX_W,
   2787                                   EVEX_CD8<64, CD8VF>;
   2788   defm VFNMSUB213PDZ : avx512_fma3p_rm<0xAE, "vfnmsub213pd", VR512, f512mem,
   2789                                   memopv8f64, f64mem, loadf64, "{1to8}",
   2790                                   X86Fnmsub, v8f64>, EVEX_V512, VEX_W,
   2791                                   EVEX_CD8<64, CD8VF>;
   2792 }
   2793 
   2794 let Constraints = "$src1 = $dst" in {
   2795 multiclass avx512_fma3p_m132<bits<8> opc, string OpcodeStr,
   2796             RegisterClass RC, X86MemOperand x86memop,
   2797             PatFrag mem_frag, X86MemOperand x86scalar_mop, PatFrag scalar_mfrag,
   2798             string BrdcstStr, SDNode OpNode, ValueType OpVT> {
   2799   let mayLoad = 1 in
   2800   def m: AVX512FMA3<opc, MRMSrcMem, (outs RC:$dst),
   2801           (ins RC:$src1, RC:$src3, x86memop:$src2),
   2802           !strconcat(OpcodeStr, " \t{$src2, $src3, $dst|$dst, $src3, $src2}"),
   2803           [(set RC:$dst, (OpVT (OpNode RC:$src1, (mem_frag addr:$src2), RC:$src3)))]>;
   2804    def mb: AVX512FMA3<opc, MRMSrcMem, (outs RC:$dst),
   2805            (ins RC:$src1, RC:$src3, x86scalar_mop:$src2),
   2806            !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr,
   2807             ", $src3, $dst|$dst, $src3, ${src2}", BrdcstStr, "}"),
   2808            [(set RC:$dst, (OpNode RC:$src1, 
   2809            (OpVT (X86VBroadcast (scalar_mfrag addr:$src2))), RC:$src3))]>, EVEX_B;
   2810 }
   2811 } // Constraints = "$src1 = $dst"
   2812 
   2813 
   2814 let ExeDomain = SSEPackedSingle in {
   2815   defm VFMADD132PSZ    : avx512_fma3p_m132<0x98, "vfmadd132ps", VR512, f512mem,
   2816                                     memopv16f32, f32mem, loadf32, "{1to16}",
   2817                                     X86Fmadd, v16f32>, EVEX_V512,
   2818                                     EVEX_CD8<32, CD8VF>;
   2819   defm VFMSUB132PSZ    : avx512_fma3p_m132<0x9A, "vfmsub132ps", VR512, f512mem,
   2820                                     memopv16f32, f32mem, loadf32, "{1to16}",
   2821                                     X86Fmsub, v16f32>, EVEX_V512,
   2822                                     EVEX_CD8<32, CD8VF>;
   2823   defm VFMADDSUB132PSZ : avx512_fma3p_m132<0x96, "vfmaddsub132ps", VR512, f512mem,
   2824                                     memopv16f32, f32mem, loadf32, "{1to16}",
   2825                                     X86Fmaddsub, v16f32>,
   2826                                     EVEX_V512, EVEX_CD8<32, CD8VF>;
   2827   defm VFMSUBADD132PSZ : avx512_fma3p_m132<0x97, "vfmsubadd132ps", VR512, f512mem,
   2828                                     memopv16f32, f32mem, loadf32, "{1to16}",
   2829                                     X86Fmsubadd, v16f32>,
   2830                                     EVEX_V512, EVEX_CD8<32, CD8VF>;
   2831   defm VFNMADD132PSZ   : avx512_fma3p_m132<0x9C, "vfnmadd132ps", VR512, f512mem,
   2832                                     memopv16f32, f32mem, loadf32, "{1to16}",
   2833                                     X86Fnmadd, v16f32>, EVEX_V512,
   2834                                     EVEX_CD8<32, CD8VF>;
   2835   defm VFNMSUB132PSZ   : avx512_fma3p_m132<0x9E, "vfnmsub132ps", VR512, f512mem,
   2836                                     memopv16f32, f32mem, loadf32, "{1to16}",
   2837                                     X86Fnmsub, v16f32>, EVEX_V512,
   2838                                     EVEX_CD8<32, CD8VF>;
   2839 }
   2840 let ExeDomain = SSEPackedDouble in {
   2841   defm VFMADD132PDZ    : avx512_fma3p_m132<0x98, "vfmadd132pd", VR512, f512mem,
   2842                                     memopv8f64, f64mem, loadf64, "{1to8}",
   2843                                     X86Fmadd, v8f64>, EVEX_V512,
   2844                                     VEX_W, EVEX_CD8<64, CD8VF>;
   2845   defm VFMSUB132PDZ    : avx512_fma3p_m132<0x9A, "vfmsub132pd", VR512, f512mem,
   2846                                     memopv8f64, f64mem, loadf64, "{1to8}",
   2847                                     X86Fmsub, v8f64>, EVEX_V512, VEX_W,
   2848                                     EVEX_CD8<64, CD8VF>;
   2849   defm VFMADDSUB132PDZ : avx512_fma3p_m132<0x96, "vfmaddsub132pd", VR512, f512mem,
   2850                                     memopv8f64, f64mem, loadf64, "{1to8}",
   2851                                     X86Fmaddsub, v8f64>, EVEX_V512, VEX_W,
   2852                                     EVEX_CD8<64, CD8VF>;
   2853   defm VFMSUBADD132PDZ : avx512_fma3p_m132<0x97, "vfmsubadd132pd", VR512, f512mem,
   2854                                     memopv8f64, f64mem, loadf64, "{1to8}",
   2855                                     X86Fmsubadd, v8f64>, EVEX_V512, VEX_W,
   2856                                     EVEX_CD8<64, CD8VF>;
   2857   defm VFNMADD132PDZ : avx512_fma3p_m132<0x9C, "vfnmadd132pd", VR512, f512mem,
   2858                                   memopv8f64, f64mem, loadf64, "{1to8}",
   2859                                   X86Fnmadd, v8f64>, EVEX_V512, VEX_W,
   2860                                   EVEX_CD8<64, CD8VF>;
   2861   defm VFNMSUB132PDZ : avx512_fma3p_m132<0x9E, "vfnmsub132pd", VR512, f512mem,
   2862                                   memopv8f64, f64mem, loadf64, "{1to8}",
   2863                                   X86Fnmsub, v8f64>, EVEX_V512, VEX_W,
   2864                                   EVEX_CD8<64, CD8VF>;
   2865 }
   2866 
   2867 // Scalar FMA
   2868 let Constraints = "$src1 = $dst" in {
   2869 multiclass avx512_fma3s_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, 
   2870                  RegisterClass RC, ValueType OpVT, 
   2871                  X86MemOperand x86memop, Operand memop, 
   2872                  PatFrag mem_frag> {
   2873   let isCommutable = 1 in
   2874   def r     : AVX512FMA3<opc, MRMSrcReg, (outs RC:$dst),
   2875                    (ins RC:$src1, RC:$src2, RC:$src3),
   2876                    !strconcat(OpcodeStr,
   2877                               " \t{$src3, $src2, $dst|$dst, $src2, $src3}"),
   2878                    [(set RC:$dst,
   2879                      (OpVT (OpNode RC:$src2, RC:$src1, RC:$src3)))]>;
   2880   let mayLoad = 1 in
   2881   def m     : AVX512FMA3<opc, MRMSrcMem, (outs RC:$dst),
   2882                    (ins RC:$src1, RC:$src2, f128mem:$src3),
   2883                    !strconcat(OpcodeStr,
   2884                               " \t{$src3, $src2, $dst|$dst, $src2, $src3}"),
   2885                    [(set RC:$dst,
   2886                      (OpVT (OpNode RC:$src2, RC:$src1,
   2887                             (mem_frag addr:$src3))))]>;
   2888 }
   2889 
   2890 } // Constraints = "$src1 = $dst"
   2891 
   2892 defm VFMADDSSZ  : avx512_fma3s_rm<0xA9, "vfmadd213ss", X86Fmadd, FR32X,
   2893                       f32, f32mem, ssmem, loadf32>, EVEX_CD8<32, CD8VT1>;
   2894 defm VFMADDSDZ  : avx512_fma3s_rm<0xA9, "vfmadd213sd", X86Fmadd, FR64X,
   2895                       f64, f64mem, sdmem, loadf64>, VEX_W, EVEX_CD8<64, CD8VT1>;
   2896 defm VFMSUBSSZ  : avx512_fma3s_rm<0xAB, "vfmsub213ss", X86Fmsub, FR32X,
   2897                       f32, f32mem, ssmem, loadf32>, EVEX_CD8<32, CD8VT1>;
   2898 defm VFMSUBSDZ  : avx512_fma3s_rm<0xAB, "vfmsub213sd", X86Fmsub, FR64X,
   2899                       f64, f64mem, sdmem, loadf64>, VEX_W, EVEX_CD8<64, CD8VT1>;
   2900 defm VFNMADDSSZ  : avx512_fma3s_rm<0xAD, "vfnmadd213ss", X86Fnmadd, FR32X,
   2901                       f32, f32mem, ssmem, loadf32>, EVEX_CD8<32, CD8VT1>;
   2902 defm VFNMADDSDZ  : avx512_fma3s_rm<0xAD, "vfnmadd213sd", X86Fnmadd, FR64X,
   2903                       f64, f64mem, sdmem, loadf64>, VEX_W, EVEX_CD8<64, CD8VT1>;
   2904 defm VFNMSUBSSZ  : avx512_fma3s_rm<0xAF, "vfnmsub213ss", X86Fnmsub, FR32X,
   2905                       f32, f32mem, ssmem, loadf32>, EVEX_CD8<32, CD8VT1>;
   2906 defm VFNMSUBSDZ  : avx512_fma3s_rm<0xAF, "vfnmsub213sd", X86Fnmsub, FR64X,
   2907                       f64, f64mem, sdmem, loadf64>, VEX_W, EVEX_CD8<64, CD8VT1>;
   2908 
   2909 //===----------------------------------------------------------------------===//
   2910 // AVX-512  Scalar convert from sign integer to float/double
   2911 //===----------------------------------------------------------------------===//
   2912 
   2913 multiclass avx512_vcvtsi<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC,
   2914                           X86MemOperand x86memop, string asm> {
   2915 let hasSideEffects = 0 in {
   2916   def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins DstRC:$src1, SrcRC:$src),
   2917               !strconcat(asm," \t{$src, $src1, $dst|$dst, $src1, $src}"), []>,
   2918               EVEX_4V;
   2919   let mayLoad = 1 in
   2920   def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst),
   2921               (ins DstRC:$src1, x86memop:$src),
   2922               !strconcat(asm," \t{$src, $src1, $dst|$dst, $src1, $src}"), []>,
   2923               EVEX_4V;
   2924 } // hasSideEffects = 0
   2925 }
   2926 let Predicates = [HasAVX512] in {
   2927 defm VCVTSI2SSZ   : avx512_vcvtsi<0x2A, GR32, FR32X, i32mem, "cvtsi2ss{l}">,
   2928                                   XS, VEX_LIG, EVEX_CD8<32, CD8VT1>;
   2929 defm VCVTSI642SSZ : avx512_vcvtsi<0x2A, GR64, FR32X, i64mem, "cvtsi2ss{q}">,
   2930                                   XS, VEX_W, VEX_LIG, EVEX_CD8<64, CD8VT1>;
   2931 defm VCVTSI2SDZ   : avx512_vcvtsi<0x2A, GR32, FR64X, i32mem, "cvtsi2sd{l}">,
   2932                                   XD, VEX_LIG, EVEX_CD8<32, CD8VT1>;
   2933 defm VCVTSI642SDZ : avx512_vcvtsi<0x2A, GR64, FR64X, i64mem, "cvtsi2sd{q}">,
   2934                                   XD, VEX_W, VEX_LIG, EVEX_CD8<64, CD8VT1>;
   2935 
   2936 def : Pat<(f32 (sint_to_fp (loadi32 addr:$src))),
   2937           (VCVTSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
   2938 def : Pat<(f32 (sint_to_fp (loadi64 addr:$src))),
   2939           (VCVTSI642SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
   2940 def : Pat<(f64 (sint_to_fp (loadi32 addr:$src))),
   2941           (VCVTSI2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;
   2942 def : Pat<(f64 (sint_to_fp (loadi64 addr:$src))),
   2943           (VCVTSI642SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;
   2944 
   2945 def : Pat<(f32 (sint_to_fp GR32:$src)),
   2946           (VCVTSI2SSZrr (f32 (IMPLICIT_DEF)), GR32:$src)>;
   2947 def : Pat<(f32 (sint_to_fp GR64:$src)),
   2948           (VCVTSI642SSZrr (f32 (IMPLICIT_DEF)), GR64:$src)>;
   2949 def : Pat<(f64 (sint_to_fp GR32:$src)),
   2950           (VCVTSI2SDZrr (f64 (IMPLICIT_DEF)), GR32:$src)>;
   2951 def : Pat<(f64 (sint_to_fp GR64:$src)),
   2952           (VCVTSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>;
   2953 
   2954 defm VCVTUSI2SSZ   : avx512_vcvtsi<0x7B, GR32, FR32X, i32mem, "cvtusi2ss{l}">,
   2955                                   XS, VEX_LIG, EVEX_CD8<32, CD8VT1>;
   2956 defm VCVTUSI642SSZ : avx512_vcvtsi<0x7B, GR64, FR32X, i64mem, "cvtusi2ss{q}">,
   2957                                   XS, VEX_W, VEX_LIG, EVEX_CD8<64, CD8VT1>;
   2958 defm VCVTUSI2SDZ   : avx512_vcvtsi<0x7B, GR32, FR64X, i32mem, "cvtusi2sd{l}">,
   2959                                   XD, VEX_LIG, EVEX_CD8<32, CD8VT1>;
   2960 defm VCVTUSI642SDZ : avx512_vcvtsi<0x7B, GR64, FR64X, i64mem, "cvtusi2sd{q}">,
   2961                                   XD, VEX_W, VEX_LIG, EVEX_CD8<64, CD8VT1>;
   2962 
   2963 def : Pat<(f32 (uint_to_fp (loadi32 addr:$src))),
   2964           (VCVTUSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
   2965 def : Pat<(f32 (uint_to_fp (loadi64 addr:$src))),
   2966           (VCVTUSI642SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
   2967 def : Pat<(f64 (uint_to_fp (loadi32 addr:$src))),
   2968           (VCVTUSI2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;
   2969 def : Pat<(f64 (uint_to_fp (loadi64 addr:$src))),
   2970           (VCVTUSI642SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;
   2971 
   2972 def : Pat<(f32 (uint_to_fp GR32:$src)),
   2973           (VCVTUSI2SSZrr (f32 (IMPLICIT_DEF)), GR32:$src)>;
   2974 def : Pat<(f32 (uint_to_fp GR64:$src)),
   2975           (VCVTUSI642SSZrr (f32 (IMPLICIT_DEF)), GR64:$src)>;
   2976 def : Pat<(f64 (uint_to_fp GR32:$src)),
   2977           (VCVTUSI2SDZrr (f64 (IMPLICIT_DEF)), GR32:$src)>;
   2978 def : Pat<(f64 (uint_to_fp GR64:$src)),
   2979           (VCVTUSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>;
   2980 }
   2981 
   2982 //===----------------------------------------------------------------------===//
   2983 // AVX-512  Scalar convert from float/double to integer
   2984 //===----------------------------------------------------------------------===//
   2985 multiclass avx512_cvt_s_int<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC,
   2986                           Intrinsic Int, Operand memop, ComplexPattern mem_cpat,
   2987                           string asm> {
   2988 let hasSideEffects = 0 in {
   2989   def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src),
   2990               !strconcat(asm," \t{$src, $dst|$dst, $src}"),
   2991               [(set DstRC:$dst, (Int SrcRC:$src))]>, EVEX, VEX_LIG,
   2992               Requires<[HasAVX512]>;
   2993   let mayLoad = 1 in
   2994   def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins memop:$src),
   2995               !strconcat(asm," \t{$src, $dst|$dst, $src}"), []>, EVEX, VEX_LIG,
   2996               Requires<[HasAVX512]>;
   2997 } // hasSideEffects = 0
   2998 }
   2999 let Predicates = [HasAVX512] in {
   3000 // Convert float/double to signed/unsigned int 32/64
   3001 defm VCVTSS2SIZ:    avx512_cvt_s_int<0x2D, VR128X, GR32, int_x86_sse_cvtss2si,
   3002                                    ssmem, sse_load_f32, "cvtss2si">,
   3003                                    XS, EVEX_CD8<32, CD8VT1>;
   3004 defm VCVTSS2SI64Z:  avx512_cvt_s_int<0x2D, VR128X, GR64, int_x86_sse_cvtss2si64,
   3005                                    ssmem, sse_load_f32, "cvtss2si">,
   3006                                    XS, VEX_W, EVEX_CD8<32, CD8VT1>;
   3007 defm VCVTSS2USIZ:   avx512_cvt_s_int<0x79, VR128X, GR32, int_x86_avx512_cvtss2usi,
   3008                                    ssmem, sse_load_f32, "cvtss2usi">,
   3009                                    XS, EVEX_CD8<32, CD8VT1>;
   3010 defm VCVTSS2USI64Z: avx512_cvt_s_int<0x79, VR128X, GR64,
   3011                                    int_x86_avx512_cvtss2usi64, ssmem,
   3012                                    sse_load_f32, "cvtss2usi">, XS, VEX_W,
   3013                                    EVEX_CD8<32, CD8VT1>;
   3014 defm VCVTSD2SIZ:    avx512_cvt_s_int<0x2D, VR128X, GR32, int_x86_sse2_cvtsd2si,
   3015                                    sdmem, sse_load_f64, "cvtsd2si">,
   3016                                    XD, EVEX_CD8<64, CD8VT1>;
   3017 defm VCVTSD2SI64Z:  avx512_cvt_s_int<0x2D, VR128X, GR64, int_x86_sse2_cvtsd2si64,
   3018                                    sdmem, sse_load_f64, "cvtsd2si">,
   3019                                    XD, VEX_W, EVEX_CD8<64, CD8VT1>;
   3020 defm VCVTSD2USIZ:   avx512_cvt_s_int<0x79, VR128X, GR32, int_x86_avx512_cvtsd2usi,
   3021                                    sdmem, sse_load_f64, "cvtsd2usi">,
   3022                                    XD, EVEX_CD8<64, CD8VT1>;
   3023 defm VCVTSD2USI64Z: avx512_cvt_s_int<0x79, VR128X, GR64,
   3024                                    int_x86_avx512_cvtsd2usi64, sdmem,
   3025                                    sse_load_f64, "cvtsd2usi">, XD, VEX_W,
   3026                                    EVEX_CD8<64, CD8VT1>;
   3027 
   3028 let isCodeGenOnly = 1 in {
   3029   defm Int_VCVTSI2SSZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X,
   3030             int_x86_sse_cvtsi2ss, i32mem, loadi32, "cvtsi2ss{l}",
   3031             SSE_CVT_Scalar, 0>, XS, EVEX_4V;
   3032   defm Int_VCVTSI2SS64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X,
   3033             int_x86_sse_cvtsi642ss, i64mem, loadi64, "cvtsi2ss{q}",
   3034             SSE_CVT_Scalar, 0>, XS, EVEX_4V, VEX_W;
   3035   defm Int_VCVTSI2SDZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X,
   3036             int_x86_sse2_cvtsi2sd, i32mem, loadi32, "cvtsi2sd{l}",
   3037             SSE_CVT_Scalar, 0>, XD, EVEX_4V;
   3038   defm Int_VCVTSI2SD64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X,
   3039             int_x86_sse2_cvtsi642sd, i64mem, loadi64, "cvtsi2sd{q}",
   3040             SSE_CVT_Scalar, 0>, XD, EVEX_4V, VEX_W;
   3041 
   3042   defm Int_VCVTUSI2SSZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X,
   3043             int_x86_avx512_cvtusi2ss, i32mem, loadi32, "cvtusi2ss{l}",
   3044             SSE_CVT_Scalar, 0>, XS, EVEX_4V;
   3045   defm Int_VCVTUSI2SS64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X,
   3046             int_x86_avx512_cvtusi642ss, i64mem, loadi64, "cvtusi2ss{q}",
   3047             SSE_CVT_Scalar, 0>, XS, EVEX_4V, VEX_W;
   3048   defm Int_VCVTUSI2SDZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X,
   3049             int_x86_avx512_cvtusi2sd, i32mem, loadi32, "cvtusi2sd{l}",
   3050             SSE_CVT_Scalar, 0>, XD, EVEX_4V;
   3051   defm Int_VCVTUSI2SD64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X,
   3052             int_x86_avx512_cvtusi642sd, i64mem, loadi64, "cvtusi2sd{q}",
   3053             SSE_CVT_Scalar, 0>, XD, EVEX_4V, VEX_W;
   3054 } // isCodeGenOnly = 1
   3055 
   3056 // Convert float/double to signed/unsigned int 32/64 with truncation
   3057 let isCodeGenOnly = 1 in {
   3058   defm Int_VCVTTSS2SIZ : avx512_cvt_s_int<0x2C, VR128X, GR32, int_x86_sse_cvttss2si,
   3059                                      ssmem, sse_load_f32, "cvttss2si">,
   3060                                      XS, EVEX_CD8<32, CD8VT1>;
   3061   defm Int_VCVTTSS2SI64Z : avx512_cvt_s_int<0x2C, VR128X, GR64,
   3062                                      int_x86_sse_cvttss2si64, ssmem, sse_load_f32,
   3063                                      "cvttss2si">, XS, VEX_W,
   3064                                      EVEX_CD8<32, CD8VT1>;
   3065   defm Int_VCVTTSD2SIZ : avx512_cvt_s_int<0x2C, VR128X, GR32, int_x86_sse2_cvttsd2si,
   3066                                      sdmem, sse_load_f64, "cvttsd2si">, XD,
   3067                                      EVEX_CD8<64, CD8VT1>;
   3068   defm Int_VCVTTSD2SI64Z : avx512_cvt_s_int<0x2C, VR128X, GR64,
   3069                                      int_x86_sse2_cvttsd2si64, sdmem, sse_load_f64,
   3070                                      "cvttsd2si">, XD, VEX_W,
   3071                                      EVEX_CD8<64, CD8VT1>;
   3072   defm Int_VCVTTSS2USIZ : avx512_cvt_s_int<0x78, VR128X, GR32,
   3073                                      int_x86_avx512_cvttss2usi, ssmem, sse_load_f32,
   3074                                      "cvttss2usi">, XS, EVEX_CD8<32, CD8VT1>;
   3075   defm Int_VCVTTSS2USI64Z : avx512_cvt_s_int<0x78, VR128X, GR64,
   3076                                      int_x86_avx512_cvttss2usi64, ssmem,
   3077                                      sse_load_f32, "cvttss2usi">, XS, VEX_W,
   3078                                      EVEX_CD8<32, CD8VT1>;
   3079   defm Int_VCVTTSD2USIZ : avx512_cvt_s_int<0x78, VR128X, GR32,
   3080                                      int_x86_avx512_cvttsd2usi,
   3081                                      sdmem, sse_load_f64, "cvttsd2usi">, XD,
   3082                                      EVEX_CD8<64, CD8VT1>;
   3083   defm Int_VCVTTSD2USI64Z : avx512_cvt_s_int<0x78, VR128X, GR64,
   3084                                      int_x86_avx512_cvttsd2usi64, sdmem,
   3085                                      sse_load_f64, "cvttsd2usi">, XD, VEX_W,
   3086                                      EVEX_CD8<64, CD8VT1>;
   3087 } // isCodeGenOnly = 1
   3088 
   3089 multiclass avx512_cvt_s<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC,
   3090                          SDNode OpNode, X86MemOperand x86memop, PatFrag ld_frag,
   3091                          string asm> {
   3092   def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src),
   3093               !strconcat(asm," \t{$src, $dst|$dst, $src}"),
   3094               [(set DstRC:$dst, (OpNode SrcRC:$src))]>, EVEX;
   3095   def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src),
   3096               !strconcat(asm," \t{$src, $dst|$dst, $src}"),
   3097               [(set DstRC:$dst, (OpNode (ld_frag addr:$src)))]>, EVEX;
   3098 }
   3099 
   3100 defm VCVTTSS2SIZ    : avx512_cvt_s<0x2C, FR32X, GR32, fp_to_sint, f32mem,
   3101                                   loadf32, "cvttss2si">, XS,
   3102                                   EVEX_CD8<32, CD8VT1>;
   3103 defm VCVTTSS2USIZ   : avx512_cvt_s<0x78, FR32X, GR32, fp_to_uint, f32mem,
   3104                                   loadf32, "cvttss2usi">, XS,
   3105                                   EVEX_CD8<32, CD8VT1>;
   3106 defm VCVTTSS2SI64Z  : avx512_cvt_s<0x2C, FR32X, GR64, fp_to_sint, f32mem,
   3107                                   loadf32, "cvttss2si">, XS, VEX_W,
   3108                                   EVEX_CD8<32, CD8VT1>;
   3109 defm VCVTTSS2USI64Z : avx512_cvt_s<0x78, FR32X, GR64, fp_to_uint, f32mem,
   3110                                   loadf32, "cvttss2usi">, XS, VEX_W,
   3111                                   EVEX_CD8<32, CD8VT1>;
   3112 defm VCVTTSD2SIZ    : avx512_cvt_s<0x2C, FR64X, GR32, fp_to_sint, f64mem,
   3113                                   loadf64, "cvttsd2si">, XD,
   3114                                   EVEX_CD8<64, CD8VT1>;
   3115 defm VCVTTSD2USIZ   : avx512_cvt_s<0x78, FR64X, GR32, fp_to_uint, f64mem,
   3116                                   loadf64, "cvttsd2usi">, XD,
   3117                                   EVEX_CD8<64, CD8VT1>;
   3118 defm VCVTTSD2SI64Z  : avx512_cvt_s<0x2C, FR64X, GR64, fp_to_sint, f64mem,
   3119                                   loadf64, "cvttsd2si">, XD, VEX_W,
   3120                                   EVEX_CD8<64, CD8VT1>;
   3121 defm VCVTTSD2USI64Z : avx512_cvt_s<0x78, FR64X, GR64, fp_to_uint, f64mem,
   3122                                   loadf64, "cvttsd2usi">, XD, VEX_W,
   3123                                   EVEX_CD8<64, CD8VT1>;
   3124 } // HasAVX512
   3125 //===----------------------------------------------------------------------===//
   3126 // AVX-512  Convert form float to double and back
   3127 //===----------------------------------------------------------------------===//
   3128 let hasSideEffects = 0 in {
   3129 def VCVTSS2SDZrr : AVX512XSI<0x5A, MRMSrcReg, (outs FR64X:$dst),
   3130                     (ins FR32X:$src1, FR32X:$src2),
   3131                     "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
   3132                     []>, EVEX_4V, VEX_LIG, Sched<[WriteCvtF2F]>;
   3133 let mayLoad = 1 in
   3134 def VCVTSS2SDZrm : AVX512XSI<0x5A, MRMSrcMem, (outs FR64X:$dst),
   3135                     (ins FR32X:$src1, f32mem:$src2),
   3136                     "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
   3137                     []>, EVEX_4V, VEX_LIG, Sched<[WriteCvtF2FLd, ReadAfterLd]>,
   3138                     EVEX_CD8<32, CD8VT1>;
   3139 
   3140 // Convert scalar double to scalar single
   3141 def VCVTSD2SSZrr  : AVX512XDI<0x5A, MRMSrcReg, (outs FR32X:$dst),
   3142                       (ins FR64X:$src1, FR64X:$src2),
   3143                       "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}",
   3144                       []>, EVEX_4V, VEX_LIG, VEX_W, Sched<[WriteCvtF2F]>;
   3145 let mayLoad = 1 in
   3146 def VCVTSD2SSZrm  : AVX512XDI<0x5A, MRMSrcMem, (outs FR32X:$dst),
   3147                       (ins FR64X:$src1, f64mem:$src2),
   3148                       "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}",
   3149                       []>, EVEX_4V, VEX_LIG, VEX_W,
   3150                       Sched<[WriteCvtF2FLd, ReadAfterLd]>, EVEX_CD8<64, CD8VT1>;
   3151 }
   3152 
   3153 def : Pat<(f64 (fextend FR32X:$src)), (VCVTSS2SDZrr FR32X:$src, FR32X:$src)>,
   3154       Requires<[HasAVX512]>;
   3155 def : Pat<(fextend (loadf32 addr:$src)),
   3156     (VCVTSS2SDZrm (f32 (IMPLICIT_DEF)), addr:$src)>, Requires<[HasAVX512]>;
   3157 
   3158 def : Pat<(extloadf32 addr:$src),
   3159     (VCVTSS2SDZrm (f32 (IMPLICIT_DEF)), addr:$src)>,
   3160       Requires<[HasAVX512, OptForSize]>;
   3161 
   3162 def : Pat<(extloadf32 addr:$src),
   3163     (VCVTSS2SDZrr (f32 (IMPLICIT_DEF)), (VMOVSSZrm addr:$src))>,
   3164     Requires<[HasAVX512, OptForSpeed]>;
   3165 
   3166 def : Pat<(f32 (fround FR64X:$src)), (VCVTSD2SSZrr FR64X:$src, FR64X:$src)>,
   3167            Requires<[HasAVX512]>;
   3168 
   3169 multiclass avx512_vcvt_fp_with_rc<bits<8> opc, string asm, RegisterClass SrcRC, 
   3170                RegisterClass DstRC, SDNode OpNode, PatFrag mem_frag, 
   3171                X86MemOperand x86memop, ValueType OpVT, ValueType InVT,
   3172                Domain d> {
   3173 let hasSideEffects = 0 in {
   3174   def rr : AVX512PI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src),
   3175               !strconcat(asm," \t{$src, $dst|$dst, $src}"),
   3176               [(set DstRC:$dst,
   3177                 (OpVT (OpNode (InVT SrcRC:$src))))], d>, EVEX;
   3178   def rrb : AVX512PI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src, AVX512RC:$rc),
   3179               !strconcat(asm," \t{$rc, $src, $dst|$dst, $src, $rc}"),
   3180               [], d>, EVEX, EVEX_B, EVEX_RC;
   3181   let mayLoad = 1 in
   3182   def rm : AVX512PI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src),
   3183               !strconcat(asm," \t{$src, $dst|$dst, $src}"),
   3184               [(set DstRC:$dst,
   3185                 (OpVT (OpNode (InVT (bitconvert (mem_frag addr:$src))))))], d>, EVEX;
   3186 } // hasSideEffects = 0
   3187 }
   3188 
   3189 multiclass avx512_vcvt_fp<bits<8> opc, string asm, RegisterClass SrcRC,
   3190                RegisterClass DstRC, SDNode OpNode, PatFrag mem_frag,
   3191                X86MemOperand x86memop, ValueType OpVT, ValueType InVT,
   3192                Domain d> {
   3193 let hasSideEffects = 0 in {
   3194   def rr : AVX512PI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src),
   3195               !strconcat(asm," \t{$src, $dst|$dst, $src}"),
   3196               [(set DstRC:$dst,
   3197                 (OpVT (OpNode (InVT SrcRC:$src))))], d>, EVEX;
   3198   let mayLoad = 1 in
   3199   def rm : AVX512PI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src),
   3200               !strconcat(asm," \t{$src, $dst|$dst, $src}"),
   3201               [(set DstRC:$dst,
   3202                 (OpVT (OpNode (InVT (bitconvert (mem_frag addr:$src))))))], d>, EVEX;
   3203 } // hasSideEffects = 0
   3204 }
   3205 
   3206 defm VCVTPD2PSZ : avx512_vcvt_fp_with_rc<0x5A, "vcvtpd2ps", VR512, VR256X, fround,
   3207                                 memopv8f64, f512mem, v8f32, v8f64,
   3208                                 SSEPackedSingle>, EVEX_V512, VEX_W, PD,
   3209                                 EVEX_CD8<64, CD8VF>;
   3210 
   3211 defm VCVTPS2PDZ : avx512_vcvt_fp<0x5A, "vcvtps2pd", VR256X, VR512, fextend,
   3212                                 memopv4f64, f256mem, v8f64, v8f32,
   3213                                 SSEPackedDouble>, EVEX_V512, PS,
   3214                                 EVEX_CD8<32, CD8VH>;
   3215 def : Pat<(v8f64 (extloadv8f32 addr:$src)),
   3216             (VCVTPS2PDZrm addr:$src)>;
   3217             
   3218 def : Pat<(v8f32 (int_x86_avx512_mask_cvtpd2ps_512 (v8f64 VR512:$src),
   3219                    (bc_v8f32(v8i32 immAllZerosV)), (i8 -1), (i32 FROUND_CURRENT))),
   3220           (VCVTPD2PSZrr VR512:$src)>;
   3221 
   3222 def : Pat<(v8f32 (int_x86_avx512_mask_cvtpd2ps_512 (v8f64 VR512:$src),
   3223                    (bc_v8f32(v8i32 immAllZerosV)), (i8 -1), imm:$rc)),
   3224           (VCVTPD2PSZrrb VR512:$src, imm:$rc)>;
   3225 
   3226 //===----------------------------------------------------------------------===//
   3227 // AVX-512  Vector convert from sign integer to float/double
   3228 //===----------------------------------------------------------------------===//
   3229 
   3230 defm VCVTDQ2PSZ : avx512_vcvt_fp_with_rc<0x5B, "vcvtdq2ps", VR512, VR512, sint_to_fp,
   3231                                 memopv8i64, i512mem, v16f32, v16i32,
   3232                                 SSEPackedSingle>, EVEX_V512, PS,
   3233                                 EVEX_CD8<32, CD8VF>;
   3234 
   3235 defm VCVTDQ2PDZ : avx512_vcvt_fp<0xE6, "vcvtdq2pd", VR256X, VR512, sint_to_fp,
   3236                                 memopv4i64, i256mem, v8f64, v8i32,
   3237                                 SSEPackedDouble>, EVEX_V512, XS,
   3238                                 EVEX_CD8<32, CD8VH>;
   3239 
   3240 defm VCVTTPS2DQZ : avx512_vcvt_fp<0x5B, "vcvttps2dq", VR512, VR512, fp_to_sint,
   3241                                  memopv16f32, f512mem, v16i32, v16f32,
   3242                                  SSEPackedSingle>, EVEX_V512, XS,
   3243                                  EVEX_CD8<32, CD8VF>;
   3244 
   3245 defm VCVTTPD2DQZ : avx512_vcvt_fp<0xE6, "vcvttpd2dq", VR512, VR256X, fp_to_sint,
   3246                                  memopv8f64, f512mem, v8i32, v8f64, 
   3247                                  SSEPackedDouble>, EVEX_V512, PD, VEX_W,
   3248                                  EVEX_CD8<64, CD8VF>;
   3249 
   3250 defm VCVTTPS2UDQZ : avx512_vcvt_fp<0x78, "vcvttps2udq", VR512, VR512, fp_to_uint,
   3251                                  memopv16f32, f512mem, v16i32, v16f32,
   3252                                  SSEPackedSingle>, EVEX_V512, PS,
   3253                                  EVEX_CD8<32, CD8VF>;
   3254 
   3255 // cvttps2udq (src, 0, mask-all-ones, sae-current)
   3256 def : Pat<(v16i32 (int_x86_avx512_mask_cvttps2udq_512 (v16f32 VR512:$src),
   3257                    (v16i32 immAllZerosV), (i16 -1), FROUND_CURRENT)),
   3258           (VCVTTPS2UDQZrr VR512:$src)>;
   3259 
   3260 defm VCVTTPD2UDQZ : avx512_vcvt_fp<0x78, "vcvttpd2udq", VR512, VR256X, fp_to_uint,
   3261                                  memopv8f64, f512mem, v8i32, v8f64,
   3262                                  SSEPackedDouble>, EVEX_V512, PS, VEX_W,
   3263                                  EVEX_CD8<64, CD8VF>;
   3264                                  
   3265 // cvttpd2udq (src, 0, mask-all-ones, sae-current)
   3266 def : Pat<(v8i32 (int_x86_avx512_mask_cvttpd2udq_512 (v8f64 VR512:$src),
   3267                    (v8i32 immAllZerosV), (i8 -1), FROUND_CURRENT)),
   3268           (VCVTTPD2UDQZrr VR512:$src)>;
   3269 
   3270 defm VCVTUDQ2PDZ : avx512_vcvt_fp<0x7A, "vcvtudq2pd", VR256X, VR512, uint_to_fp,
   3271                                  memopv4i64, f256mem, v8f64, v8i32,
   3272                                  SSEPackedDouble>, EVEX_V512, XS,
   3273                                  EVEX_CD8<32, CD8VH>;
   3274                                  
   3275 defm VCVTUDQ2PSZ : avx512_vcvt_fp_with_rc<0x7A, "vcvtudq2ps", VR512, VR512, uint_to_fp,
   3276                                  memopv16i32, f512mem, v16f32, v16i32,
   3277                                  SSEPackedSingle>, EVEX_V512, XD,
   3278                                  EVEX_CD8<32, CD8VF>;
   3279 
   3280 def : Pat<(v8i32 (fp_to_uint (v8f32 VR256X:$src1))),
   3281           (EXTRACT_SUBREG (v16i32 (VCVTTPS2UDQZrr 
   3282            (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>;
   3283                                  
   3284 def : Pat<(v4i32 (fp_to_uint (v4f32 VR128X:$src1))),
   3285           (EXTRACT_SUBREG (v16i32 (VCVTTPS2UDQZrr
   3286            (v16f32 (SUBREG_TO_REG (i32 0), VR128X:$src1, sub_xmm)))), sub_xmm)>;
   3287 
   3288 def : Pat<(v8f32 (uint_to_fp (v8i32 VR256X:$src1))),
   3289           (EXTRACT_SUBREG (v16f32 (VCVTUDQ2PSZrr
   3290            (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>;
   3291            
   3292 def : Pat<(v4f32 (uint_to_fp (v4i32 VR128X:$src1))),
   3293           (EXTRACT_SUBREG (v16f32 (VCVTUDQ2PSZrr
   3294            (v16i32 (SUBREG_TO_REG (i32 0), VR128X:$src1, sub_xmm)))), sub_xmm)>;
   3295 
   3296 def : Pat<(v4f64 (uint_to_fp (v4i32 VR128X:$src1))),
   3297           (EXTRACT_SUBREG (v8f64 (VCVTUDQ2PDZrr
   3298            (v8i32 (SUBREG_TO_REG (i32 0), VR128X:$src1, sub_xmm)))), sub_ymm)>;
   3299 
   3300 def : Pat<(v16f32 (int_x86_avx512_mask_cvtdq2ps_512 (v16i32 VR512:$src),
   3301                    (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1), imm:$rc)),
   3302           (VCVTDQ2PSZrrb VR512:$src, imm:$rc)>;
   3303 def : Pat<(v8f64 (int_x86_avx512_mask_cvtdq2pd_512 (v8i32 VR256X:$src),
   3304                    (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))),
   3305           (VCVTDQ2PDZrr VR256X:$src)>;
   3306 def : Pat<(v16f32 (int_x86_avx512_mask_cvtudq2ps_512 (v16i32 VR512:$src),
   3307                    (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1), imm:$rc)),
   3308           (VCVTUDQ2PSZrrb VR512:$src, imm:$rc)>;
   3309 def : Pat<(v8f64 (int_x86_avx512_mask_cvtudq2pd_512 (v8i32 VR256X:$src),
   3310                    (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))),
   3311           (VCVTUDQ2PDZrr VR256X:$src)>;
   3312 
   3313 multiclass avx512_vcvt_fp2int<bits<8> opc, string asm, RegisterClass SrcRC,
   3314                RegisterClass DstRC, PatFrag mem_frag,
   3315                X86MemOperand x86memop, Domain d> {
   3316 let hasSideEffects = 0 in {
   3317   def rr : AVX512PI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src),
   3318               !strconcat(asm," \t{$src, $dst|$dst, $src}"),
   3319               [], d>, EVEX;
   3320   def rrb : AVX512PI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src, AVX512RC:$rc),
   3321               !strconcat(asm," \t{$rc, $src, $dst|$dst, $src, $rc}"),
   3322               [], d>, EVEX, EVEX_B, EVEX_RC;
   3323   let mayLoad = 1 in
   3324   def rm : AVX512PI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src),
   3325               !strconcat(asm," \t{$src, $dst|$dst, $src}"),
   3326               [], d>, EVEX;
   3327 } // hasSideEffects = 0
   3328 }
   3329 
   3330 defm VCVTPS2DQZ : avx512_vcvt_fp2int<0x5B, "vcvtps2dq", VR512, VR512,
   3331                                  memopv16f32, f512mem, SSEPackedSingle>, PD,
   3332                                  EVEX_V512, EVEX_CD8<32, CD8VF>;
   3333 defm VCVTPD2DQZ : avx512_vcvt_fp2int<0xE6, "vcvtpd2dq", VR512, VR256X,
   3334                                  memopv8f64, f512mem, SSEPackedDouble>, XD, VEX_W,
   3335                                  EVEX_V512, EVEX_CD8<64, CD8VF>;
   3336 
   3337 def : Pat <(v16i32 (int_x86_avx512_mask_cvtps2dq_512 (v16f32 VR512:$src),
   3338                     (v16i32 immAllZerosV), (i16 -1), imm:$rc)),
   3339            (VCVTPS2DQZrrb VR512:$src, imm:$rc)>;
   3340 
   3341 def : Pat <(v8i32 (int_x86_avx512_mask_cvtpd2dq_512 (v8f64 VR512:$src),
   3342                     (v8i32 immAllZerosV), (i8 -1), imm:$rc)),
   3343            (VCVTPD2DQZrrb VR512:$src, imm:$rc)>;
   3344 
   3345 defm VCVTPS2UDQZ : avx512_vcvt_fp2int<0x79, "vcvtps2udq", VR512, VR512,
   3346                                  memopv16f32, f512mem, SSEPackedSingle>,
   3347                                  PS, EVEX_V512, EVEX_CD8<32, CD8VF>;
   3348 defm VCVTPD2UDQZ : avx512_vcvt_fp2int<0x79, "vcvtpd2udq", VR512, VR256X,
   3349                                  memopv8f64, f512mem, SSEPackedDouble>, VEX_W,
   3350                                  PS, EVEX_V512, EVEX_CD8<64, CD8VF>;
   3351 
   3352 def : Pat <(v16i32 (int_x86_avx512_mask_cvtps2udq_512 (v16f32 VR512:$src),
   3353                     (v16i32 immAllZerosV), (i16 -1), imm:$rc)),
   3354            (VCVTPS2UDQZrrb VR512:$src, imm:$rc)>;
   3355 
   3356 def : Pat <(v8i32 (int_x86_avx512_mask_cvtpd2udq_512 (v8f64 VR512:$src),
   3357                     (v8i32 immAllZerosV), (i8 -1), imm:$rc)),
   3358            (VCVTPD2UDQZrrb VR512:$src, imm:$rc)>;
   3359 
   3360 let Predicates = [HasAVX512] in {
   3361   def : Pat<(v8f32 (fround (loadv8f64 addr:$src))),
   3362             (VCVTPD2PSZrm addr:$src)>;
   3363   def : Pat<(v8f64 (extloadv8f32 addr:$src)),
   3364             (VCVTPS2PDZrm addr:$src)>;
   3365 }
   3366 
   3367 //===----------------------------------------------------------------------===//
   3368 // Half precision conversion instructions
   3369 //===----------------------------------------------------------------------===//
   3370 multiclass avx512_cvtph2ps<RegisterClass destRC, RegisterClass srcRC,
   3371                              X86MemOperand x86memop> {
   3372   def rr : AVX5128I<0x13, MRMSrcReg, (outs destRC:$dst), (ins srcRC:$src),
   3373              "vcvtph2ps\t{$src, $dst|$dst, $src}",
   3374              []>, EVEX;
   3375   let hasSideEffects = 0, mayLoad = 1 in
   3376   def rm : AVX5128I<0x13, MRMSrcMem, (outs destRC:$dst), (ins x86memop:$src),
   3377              "vcvtph2ps\t{$src, $dst|$dst, $src}", []>, EVEX;
   3378 }
   3379 
   3380 multiclass avx512_cvtps2ph<RegisterClass destRC, RegisterClass srcRC,
   3381                              X86MemOperand x86memop> {
   3382   def rr : AVX512AIi8<0x1D, MRMDestReg, (outs destRC:$dst),
   3383                (ins srcRC:$src1, i32i8imm:$src2),
   3384                "vcvtps2ph \t{$src2, $src1, $dst|$dst, $src1, $src2}",
   3385                []>, EVEX;
   3386   let hasSideEffects = 0, mayStore = 1 in
   3387   def mr : AVX512AIi8<0x1D, MRMDestMem, (outs),
   3388                (ins x86memop:$dst, srcRC:$src1, i32i8imm:$src2),
   3389                "vcvtps2ph \t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, EVEX;
   3390 }
   3391 
   3392 defm VCVTPH2PSZ : avx512_cvtph2ps<VR512, VR256X, f256mem>, EVEX_V512,
   3393                                     EVEX_CD8<32, CD8VH>;
   3394 defm VCVTPS2PHZ : avx512_cvtps2ph<VR256X, VR512, f256mem>, EVEX_V512,
   3395                                     EVEX_CD8<32, CD8VH>;
   3396 
   3397 def : Pat<(v16i16 (int_x86_avx512_mask_vcvtps2ph_512 (v16f32 VR512:$src),
   3398            imm:$rc, (bc_v16i16(v8i32 immAllZerosV)), (i16 -1))),
   3399            (VCVTPS2PHZrr VR512:$src, imm:$rc)>;
   3400 
   3401 def : Pat<(v16f32 (int_x86_avx512_mask_vcvtph2ps_512 (v16i16 VR256X:$src),
   3402            (bc_v16f32(v16i32 immAllZerosV)), (i16 -1), (i32 FROUND_CURRENT))),
   3403            (VCVTPH2PSZrr VR256X:$src)>;
   3404 
   3405 let Defs = [EFLAGS], Predicates = [HasAVX512] in {
   3406   defm VUCOMISSZ : sse12_ord_cmp<0x2E, FR32X, X86cmp, f32, f32mem, loadf32,
   3407                                  "ucomiss">, PS, EVEX, VEX_LIG,
   3408                                  EVEX_CD8<32, CD8VT1>;
   3409   defm VUCOMISDZ : sse12_ord_cmp<0x2E, FR64X, X86cmp, f64, f64mem, loadf64,
   3410                                   "ucomisd">, PD, EVEX,
   3411                                   VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
   3412   let Pattern = []<dag> in {
   3413     defm VCOMISSZ  : sse12_ord_cmp<0x2F, VR128X, undef, v4f32, f128mem, load,
   3414                                    "comiss">, PS, EVEX, VEX_LIG,
   3415                                    EVEX_CD8<32, CD8VT1>;
   3416     defm VCOMISDZ  : sse12_ord_cmp<0x2F, VR128X, undef, v2f64, f128mem, load,
   3417                                    "comisd">, PD, EVEX,
   3418                                     VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
   3419   }
   3420   let isCodeGenOnly = 1 in {
   3421     defm Int_VUCOMISSZ  : sse12_ord_cmp<0x2E, VR128X, X86ucomi, v4f32, f128mem,
   3422                               load, "ucomiss">, PS, EVEX, VEX_LIG,
   3423                               EVEX_CD8<32, CD8VT1>;
   3424     defm Int_VUCOMISDZ  : sse12_ord_cmp<0x2E, VR128X, X86ucomi, v2f64, f128mem,
   3425                               load, "ucomisd">, PD, EVEX,
   3426                               VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
   3427 
   3428     defm Int_VCOMISSZ  : sse12_ord_cmp<0x2F, VR128X, X86comi, v4f32, f128mem,
   3429                               load, "comiss">, PS, EVEX, VEX_LIG,
   3430                               EVEX_CD8<32, CD8VT1>;
   3431     defm Int_VCOMISDZ  : sse12_ord_cmp<0x2F, VR128X, X86comi, v2f64, f128mem,
   3432                               load, "comisd">, PD, EVEX,
   3433                               VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
   3434   }
   3435 }
   3436   
   3437 /// avx512_fp14_s rcp14ss, rcp14sd, rsqrt14ss, rsqrt14sd
   3438 multiclass avx512_fp14_s<bits<8> opc, string OpcodeStr, RegisterClass RC,
   3439                             X86MemOperand x86memop> {
   3440   let hasSideEffects = 0 in {
   3441   def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst),
   3442                (ins RC:$src1, RC:$src2),
   3443                !strconcat(OpcodeStr,
   3444                " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, EVEX_4V;
   3445   let mayLoad = 1 in {
   3446   def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
   3447                (ins RC:$src1, x86memop:$src2),
   3448                !strconcat(OpcodeStr,
   3449                " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, EVEX_4V;
   3450   }
   3451 }
   3452 }
   3453 
   3454 defm VRCP14SS   : avx512_fp14_s<0x4D, "vrcp14ss", FR32X, f32mem>,
   3455                   EVEX_CD8<32, CD8VT1>;
   3456 defm VRCP14SD   : avx512_fp14_s<0x4D, "vrcp14sd", FR64X, f64mem>,
   3457                   VEX_W, EVEX_CD8<64, CD8VT1>;
   3458 defm VRSQRT14SS   : avx512_fp14_s<0x4F, "vrsqrt14ss", FR32X, f32mem>,
   3459                   EVEX_CD8<32, CD8VT1>;
   3460 defm VRSQRT14SD   : avx512_fp14_s<0x4F, "vrsqrt14sd", FR64X, f64mem>,
   3461                   VEX_W, EVEX_CD8<64, CD8VT1>;
   3462 
   3463 def : Pat <(v4f32 (int_x86_avx512_rcp14_ss (v4f32 VR128X:$src1),
   3464               (v4f32 VR128X:$src2), (bc_v4f32 (v4i32 immAllZerosV)), (i8 -1))),
   3465            (COPY_TO_REGCLASS (VRCP14SSrr (COPY_TO_REGCLASS VR128X:$src1, FR32X),
   3466                        (COPY_TO_REGCLASS VR128X:$src2, FR32X)), VR128X)>;
   3467 
   3468 def : Pat <(v2f64 (int_x86_avx512_rcp14_sd (v2f64 VR128X:$src1),
   3469               (v2f64 VR128X:$src2), (bc_v2f64 (v4i32 immAllZerosV)), (i8 -1))),
   3470            (COPY_TO_REGCLASS (VRCP14SDrr (COPY_TO_REGCLASS VR128X:$src1, FR64X),
   3471                        (COPY_TO_REGCLASS VR128X:$src2, FR64X)), VR128X)>;
   3472 
   3473 def : Pat <(v4f32 (int_x86_avx512_rsqrt14_ss (v4f32 VR128X:$src1),
   3474               (v4f32 VR128X:$src2), (bc_v4f32 (v4i32 immAllZerosV)), (i8 -1))),
   3475            (COPY_TO_REGCLASS (VRSQRT14SSrr (COPY_TO_REGCLASS VR128X:$src1, FR32X),
   3476                        (COPY_TO_REGCLASS VR128X:$src2, FR32X)), VR128X)>;
   3477 
   3478 def : Pat <(v2f64 (int_x86_avx512_rsqrt14_sd (v2f64 VR128X:$src1),
   3479               (v2f64 VR128X:$src2), (bc_v2f64 (v4i32 immAllZerosV)), (i8 -1))),
   3480            (COPY_TO_REGCLASS (VRSQRT14SDrr (COPY_TO_REGCLASS VR128X:$src1, FR64X),
   3481                        (COPY_TO_REGCLASS VR128X:$src2, FR64X)), VR128X)>;
   3482 
   3483 /// avx512_fp14_p rcp14ps, rcp14pd, rsqrt14ps, rsqrt14pd
   3484 multiclass avx512_fp14_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
   3485                          RegisterClass RC, X86MemOperand x86memop,
   3486                          PatFrag mem_frag, ValueType OpVt> {
   3487   def r : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src),
   3488                         !strconcat(OpcodeStr,
   3489                                    " \t{$src, $dst|$dst, $src}"),
   3490                         [(set RC:$dst, (OpVt (OpNode RC:$src)))]>,
   3491                         EVEX;
   3492   def m : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src),
   3493                         !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
   3494                         [(set RC:$dst, (OpVt (OpNode (mem_frag addr:$src))))]>,
   3495                         EVEX;
   3496 }
   3497 defm VRSQRT14PSZ : avx512_fp14_p<0x4E, "vrsqrt14ps", X86frsqrt, VR512, f512mem,
   3498                         memopv16f32, v16f32>, EVEX_V512, EVEX_CD8<32, CD8VF>;
   3499 defm VRSQRT14PDZ : avx512_fp14_p<0x4E, "vrsqrt14pd", X86frsqrt, VR512, f512mem,
   3500                         memopv8f64, v8f64>, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
   3501 defm VRCP14PSZ : avx512_fp14_p<0x4C, "vrcp14ps", X86frcp, VR512, f512mem,
   3502                         memopv16f32, v16f32>, EVEX_V512, EVEX_CD8<32, CD8VF>;
   3503 defm VRCP14PDZ : avx512_fp14_p<0x4C, "vrcp14pd", X86frcp, VR512, f512mem,
   3504                         memopv8f64, v8f64>, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
   3505 
   3506 def : Pat <(v16f32 (int_x86_avx512_rsqrt14_ps_512 (v16f32 VR512:$src),
   3507               (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1))),
   3508            (VRSQRT14PSZr VR512:$src)>;
   3509 def : Pat <(v8f64 (int_x86_avx512_rsqrt14_pd_512 (v8f64 VR512:$src),
   3510               (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))),
   3511            (VRSQRT14PDZr VR512:$src)>;
   3512 
   3513 def : Pat <(v16f32 (int_x86_avx512_rcp14_ps_512 (v16f32 VR512:$src),
   3514               (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1))),
   3515            (VRCP14PSZr VR512:$src)>;
   3516 def : Pat <(v8f64 (int_x86_avx512_rcp14_pd_512 (v8f64 VR512:$src),
   3517               (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))),
   3518            (VRCP14PDZr VR512:$src)>;
   3519 
   3520 /// avx512_fp28_s rcp28ss, rcp28sd, rsqrt28ss, rsqrt28sd
   3521 multiclass avx512_fp28_s<bits<8> opc, string OpcodeStr, RegisterClass RC,
   3522                             X86MemOperand x86memop> {
   3523   let hasSideEffects = 0, Predicates = [HasERI] in {
   3524   def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst),
   3525                (ins RC:$src1, RC:$src2),
   3526                !strconcat(OpcodeStr,
   3527                " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, EVEX_4V;
   3528   def rrb : AVX5128I<opc, MRMSrcReg, (outs RC:$dst),
   3529                (ins RC:$src1, RC:$src2),
   3530                !strconcat(OpcodeStr,
   3531                " \t{{sae}, $src2, $src1, $dst|$dst, $src1, $src2, {sae}}"),
   3532                []>, EVEX_4V, EVEX_B;
   3533   let mayLoad = 1 in {
   3534   def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
   3535                (ins RC:$src1, x86memop:$src2),
   3536                !strconcat(OpcodeStr,
   3537                " \t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, EVEX_4V;
   3538   }
   3539 }
   3540 }
   3541 
   3542 defm VRCP28SS   : avx512_fp28_s<0xCB, "vrcp28ss", FR32X, f32mem>,
   3543                   EVEX_CD8<32, CD8VT1>;
   3544 defm VRCP28SD   : avx512_fp28_s<0xCB, "vrcp28sd", FR64X, f64mem>,
   3545                   VEX_W, EVEX_CD8<64, CD8VT1>;
   3546 defm VRSQRT28SS   : avx512_fp28_s<0xCD, "vrsqrt28ss", FR32X, f32mem>,
   3547                   EVEX_CD8<32, CD8VT1>;
   3548 defm VRSQRT28SD   : avx512_fp28_s<0xCD, "vrsqrt28sd", FR64X, f64mem>,
   3549                   VEX_W, EVEX_CD8<64, CD8VT1>;
   3550 
   3551 def : Pat <(v4f32 (int_x86_avx512_rcp28_ss (v4f32 VR128X:$src1),
   3552               (v4f32 VR128X:$src2), (bc_v4f32 (v4i32 immAllZerosV)), (i8 -1),
   3553                    FROUND_NO_EXC)),
   3554            (COPY_TO_REGCLASS (VRCP28SSrrb (COPY_TO_REGCLASS VR128X:$src1, FR32X),
   3555                        (COPY_TO_REGCLASS VR128X:$src2, FR32X)), VR128X)>;
   3556 
   3557 def : Pat <(v2f64 (int_x86_avx512_rcp28_sd (v2f64 VR128X:$src1),
   3558               (v2f64 VR128X:$src2), (bc_v2f64 (v4i32 immAllZerosV)), (i8 -1),
   3559                    FROUND_NO_EXC)),
   3560            (COPY_TO_REGCLASS (VRCP28SDrrb (COPY_TO_REGCLASS VR128X:$src1, FR64X),
   3561                        (COPY_TO_REGCLASS VR128X:$src2, FR64X)), VR128X)>;
   3562 
   3563 def : Pat <(v4f32 (int_x86_avx512_rsqrt28_ss (v4f32 VR128X:$src1),
   3564               (v4f32 VR128X:$src2), (bc_v4f32 (v4i32 immAllZerosV)), (i8 -1),
   3565                    FROUND_NO_EXC)),
   3566            (COPY_TO_REGCLASS (VRSQRT28SSrrb (COPY_TO_REGCLASS VR128X:$src1, FR32X),
   3567                        (COPY_TO_REGCLASS VR128X:$src2, FR32X)), VR128X)>;
   3568 
   3569 def : Pat <(v2f64 (int_x86_avx512_rsqrt28_sd (v2f64 VR128X:$src1),
   3570               (v2f64 VR128X:$src2), (bc_v2f64 (v4i32 immAllZerosV)), (i8 -1),
   3571                    FROUND_NO_EXC)),
   3572            (COPY_TO_REGCLASS (VRSQRT28SDrrb (COPY_TO_REGCLASS VR128X:$src1, FR64X),
   3573                        (COPY_TO_REGCLASS VR128X:$src2, FR64X)), VR128X)>;
   3574 
   3575 /// avx512_fp28_p rcp28ps, rcp28pd, rsqrt28ps, rsqrt28pd
   3576 multiclass avx512_fp28_p<bits<8> opc, string OpcodeStr,
   3577                          RegisterClass RC, X86MemOperand x86memop> {
   3578   let hasSideEffects = 0, Predicates = [HasERI] in {
   3579   def r : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src),
   3580                         !strconcat(OpcodeStr,
   3581                                    " \t{$src, $dst|$dst, $src}"),
   3582                         []>, EVEX;
   3583   def rb : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src),
   3584                         !strconcat(OpcodeStr,
   3585                                    " \t{{sae}, $src, $dst|$dst, $src, {sae}}"),
   3586                         []>, EVEX, EVEX_B;
   3587   def m : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src),
   3588                         !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
   3589                         []>, EVEX;
   3590   }
   3591 }
   3592 defm VRSQRT28PSZ : avx512_fp28_p<0xCC, "vrsqrt28ps", VR512, f512mem>,
   3593                         EVEX_V512, EVEX_CD8<32, CD8VF>;
   3594 defm VRSQRT28PDZ : avx512_fp28_p<0xCC, "vrsqrt28pd", VR512, f512mem>,
   3595                         VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
   3596 defm VRCP28PSZ : avx512_fp28_p<0xCA, "vrcp28ps", VR512, f512mem>,
   3597                         EVEX_V512, EVEX_CD8<32, CD8VF>;
   3598 defm VRCP28PDZ : avx512_fp28_p<0xCA, "vrcp28pd", VR512, f512mem>,
   3599                         VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
   3600 
   3601 def : Pat <(v16f32 (int_x86_avx512_rsqrt28_ps (v16f32 VR512:$src),
   3602               (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1), FROUND_NO_EXC)),
   3603            (VRSQRT28PSZrb VR512:$src)>;
   3604 def : Pat <(v8f64 (int_x86_avx512_rsqrt28_pd (v8f64 VR512:$src),
   3605               (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1), FROUND_NO_EXC)),
   3606            (VRSQRT28PDZrb VR512:$src)>;
   3607 
   3608 def : Pat <(v16f32 (int_x86_avx512_rcp28_ps (v16f32 VR512:$src),
   3609               (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1), FROUND_NO_EXC)),
   3610            (VRCP28PSZrb VR512:$src)>;
   3611 def : Pat <(v8f64 (int_x86_avx512_rcp28_pd (v8f64 VR512:$src),
   3612               (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1), FROUND_NO_EXC)),
   3613            (VRCP28PDZrb VR512:$src)>;
   3614 
   3615 multiclass avx512_sqrt_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
   3616                               Intrinsic V16F32Int, Intrinsic V8F64Int,
   3617                               OpndItins itins_s, OpndItins itins_d> {
   3618   def PSZrr :AVX512PSI<opc, MRMSrcReg, (outs VR512:$dst), (ins VR512:$src),
   3619              !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"),
   3620              [(set VR512:$dst, (v16f32 (OpNode VR512:$src)))], itins_s.rr>,
   3621              EVEX, EVEX_V512;
   3622 
   3623   let mayLoad = 1 in
   3624   def PSZrm : AVX512PSI<opc, MRMSrcMem, (outs VR512:$dst), (ins f512mem:$src),
   3625               !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"),
   3626               [(set VR512:$dst, 
   3627                 (OpNode (v16f32 (bitconvert (memopv16f32 addr:$src)))))],
   3628               itins_s.rm>, EVEX, EVEX_V512, EVEX_CD8<32, CD8VF>;
   3629 
   3630   def PDZrr : AVX512PDI<opc, MRMSrcReg, (outs VR512:$dst), (ins VR512:$src),
   3631               !strconcat(OpcodeStr, "pd\t{$src, $dst|$dst, $src}"),
   3632               [(set VR512:$dst, (v8f64 (OpNode VR512:$src)))], itins_d.rr>,
   3633               EVEX, EVEX_V512;
   3634 
   3635   let mayLoad = 1 in
   3636     def PDZrm : AVX512PDI<opc, MRMSrcMem, (outs VR512:$dst), (ins f512mem:$src),
   3637                 !strconcat(OpcodeStr, "pd\t{$src, $dst|$dst, $src}"),
   3638                 [(set VR512:$dst, (OpNode
   3639                   (v8f64 (bitconvert (memopv16f32 addr:$src)))))],
   3640                 itins_d.rm>, EVEX, EVEX_V512, EVEX_CD8<64, CD8VF>;
   3641 
   3642 let isCodeGenOnly = 1 in {
   3643   def PSZr_Int : AVX512PSI<opc, MRMSrcReg, (outs VR512:$dst), (ins VR512:$src),
   3644                            !strconcat(OpcodeStr,
   3645                                       "ps\t{$src, $dst|$dst, $src}"),
   3646                            [(set VR512:$dst, (V16F32Int VR512:$src))]>, 
   3647                            EVEX, EVEX_V512;
   3648   def PSZm_Int : AVX512PSI<opc, MRMSrcMem, (outs VR512:$dst), (ins f512mem:$src),
   3649                           !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"),
   3650                           [(set VR512:$dst, 
   3651                            (V16F32Int (memopv16f32 addr:$src)))]>, EVEX,
   3652                           EVEX_V512, EVEX_CD8<32, CD8VF>;
   3653   def PDZr_Int : AVX512PDI<opc, MRMSrcReg, (outs VR512:$dst), (ins VR512:$src),
   3654                            !strconcat(OpcodeStr, "pd\t{$src, $dst|$dst, $src}"),
   3655                            [(set VR512:$dst, (V8F64Int VR512:$src))]>, 
   3656                            EVEX, EVEX_V512, VEX_W;
   3657   def PDZm_Int : AVX512PDI<opc, MRMSrcMem, (outs VR512:$dst), (ins f512mem:$src),
   3658                          !strconcat(OpcodeStr,
   3659                          "pd\t{$src, $dst|$dst, $src}"),
   3660                          [(set VR512:$dst, (V8F64Int (memopv8f64 addr:$src)))]>,
   3661                          EVEX, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; 
   3662 } // isCodeGenOnly = 1
   3663 }
   3664 
   3665 multiclass avx512_sqrt_scalar<bits<8> opc, string OpcodeStr,
   3666                           Intrinsic F32Int, Intrinsic F64Int,
   3667                           OpndItins itins_s, OpndItins itins_d> {
   3668   def SSZr : SI<opc, MRMSrcReg, (outs FR32X:$dst),
   3669                (ins FR32X:$src1, FR32X:$src2),
   3670                !strconcat(OpcodeStr,
   3671                           "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   3672                       [], itins_s.rr>, XS, EVEX_4V;
   3673   let isCodeGenOnly = 1 in
   3674   def SSZr_Int : SIi8<opc, MRMSrcReg, (outs VR128X:$dst),
   3675                (ins VR128X:$src1, VR128X:$src2),
   3676                !strconcat(OpcodeStr,
   3677                 "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   3678                [(set VR128X:$dst, 
   3679                  (F32Int VR128X:$src1, VR128X:$src2))],
   3680                itins_s.rr>, XS, EVEX_4V;
   3681   let mayLoad = 1 in {
   3682   def SSZm : SI<opc, MRMSrcMem, (outs FR32X:$dst),
   3683                (ins FR32X:$src1, f32mem:$src2),
   3684                !strconcat(OpcodeStr,
   3685                           "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   3686                       [], itins_s.rm>, XS, EVEX_4V, EVEX_CD8<32, CD8VT1>;
   3687   let isCodeGenOnly = 1 in
   3688   def SSZm_Int : SIi8<opc, MRMSrcMem, (outs VR128X:$dst),
   3689                    (ins VR128X:$src1, ssmem:$src2),
   3690                    !strconcat(OpcodeStr,
   3691                  "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   3692                    [(set VR128X:$dst, 
   3693                      (F32Int VR128X:$src1, sse_load_f32:$src2))],
   3694                    itins_s.rm>, XS, EVEX_4V, EVEX_CD8<32, CD8VT1>;
   3695   }
   3696   def SDZr : SI<opc, MRMSrcReg, (outs FR64X:$dst),
   3697                (ins FR64X:$src1, FR64X:$src2),
   3698                !strconcat(OpcodeStr,
   3699                           "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>,
   3700                       XD, EVEX_4V, VEX_W;
   3701   let isCodeGenOnly = 1 in
   3702   def SDZr_Int : SIi8<opc, MRMSrcReg, (outs VR128X:$dst),
   3703                (ins VR128X:$src1, VR128X:$src2),
   3704                !strconcat(OpcodeStr,
   3705                 "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   3706                [(set VR128X:$dst, 
   3707                  (F64Int VR128X:$src1, VR128X:$src2))],
   3708                itins_s.rr>, XD, EVEX_4V, VEX_W;
   3709   let mayLoad = 1 in {
   3710   def SDZm : SI<opc, MRMSrcMem, (outs FR64X:$dst),
   3711                (ins FR64X:$src1, f64mem:$src2),
   3712                !strconcat(OpcodeStr,
   3713                   "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>,
   3714                XD, EVEX_4V, VEX_W, EVEX_CD8<64, CD8VT1>;
   3715   let isCodeGenOnly = 1 in
   3716   def SDZm_Int : SIi8<opc, MRMSrcMem, (outs VR128X:$dst),
   3717                   (ins VR128X:$src1, sdmem:$src2),
   3718                    !strconcat(OpcodeStr,
   3719                   "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   3720                   [(set VR128X:$dst, 
   3721                     (F64Int VR128X:$src1, sse_load_f64:$src2))]>, 
   3722                   XD, EVEX_4V, VEX_W, EVEX_CD8<64, CD8VT1>;
   3723   }
   3724 }
   3725 
   3726 
   3727 defm VSQRT  : avx512_sqrt_scalar<0x51, "sqrt", 
   3728                 int_x86_avx512_sqrt_ss, int_x86_avx512_sqrt_sd, 
   3729                 SSE_SQRTSS, SSE_SQRTSD>,
   3730               avx512_sqrt_packed<0x51, "vsqrt", fsqrt,
   3731                 int_x86_avx512_sqrt_ps_512, int_x86_avx512_sqrt_pd_512,
   3732                 SSE_SQRTPS, SSE_SQRTPD>;
   3733 
   3734 let Predicates = [HasAVX512] in {
   3735   def : Pat<(f32 (fsqrt FR32X:$src)),
   3736             (VSQRTSSZr (f32 (IMPLICIT_DEF)), FR32X:$src)>;
   3737   def : Pat<(f32 (fsqrt (load addr:$src))),
   3738             (VSQRTSSZm (f32 (IMPLICIT_DEF)), addr:$src)>,
   3739             Requires<[OptForSize]>;
   3740   def : Pat<(f64 (fsqrt FR64X:$src)),
   3741             (VSQRTSDZr (f64 (IMPLICIT_DEF)), FR64X:$src)>;
   3742   def : Pat<(f64 (fsqrt (load addr:$src))),
   3743             (VSQRTSDZm (f64 (IMPLICIT_DEF)), addr:$src)>,
   3744             Requires<[OptForSize]>;
   3745 
   3746   def : Pat<(f32 (X86frsqrt FR32X:$src)),
   3747             (VRSQRT14SSrr (f32 (IMPLICIT_DEF)), FR32X:$src)>;
   3748   def : Pat<(f32 (X86frsqrt (load addr:$src))),
   3749             (VRSQRT14SSrm (f32 (IMPLICIT_DEF)), addr:$src)>,
   3750             Requires<[OptForSize]>;
   3751 
   3752   def : Pat<(f32 (X86frcp FR32X:$src)),
   3753             (VRCP14SSrr (f32 (IMPLICIT_DEF)), FR32X:$src)>;
   3754   def : Pat<(f32 (X86frcp (load addr:$src))),
   3755             (VRCP14SSrm (f32 (IMPLICIT_DEF)), addr:$src)>,
   3756             Requires<[OptForSize]>;
   3757 
   3758   def : Pat<(int_x86_sse_sqrt_ss VR128X:$src),
   3759             (COPY_TO_REGCLASS (VSQRTSSZr (f32 (IMPLICIT_DEF)),
   3760                                         (COPY_TO_REGCLASS VR128X:$src, FR32)),
   3761                               VR128X)>;
   3762   def : Pat<(int_x86_sse_sqrt_ss sse_load_f32:$src),
   3763             (VSQRTSSZm_Int (v4f32 (IMPLICIT_DEF)), sse_load_f32:$src)>;
   3764 
   3765   def : Pat<(int_x86_sse2_sqrt_sd VR128X:$src),
   3766             (COPY_TO_REGCLASS (VSQRTSDZr (f64 (IMPLICIT_DEF)),
   3767                                         (COPY_TO_REGCLASS VR128X:$src, FR64)),
   3768                               VR128X)>;
   3769   def : Pat<(int_x86_sse2_sqrt_sd sse_load_f64:$src),
   3770             (VSQRTSDZm_Int (v2f64 (IMPLICIT_DEF)), sse_load_f64:$src)>;
   3771 }
   3772 
   3773 
   3774 multiclass avx512_fp_unop_rm<bits<8> opcps, bits<8> opcpd, string OpcodeStr,
   3775                             X86MemOperand x86memop, RegisterClass RC,
   3776                             PatFrag mem_frag32, PatFrag mem_frag64,
   3777                             Intrinsic V4F32Int, Intrinsic V2F64Int,
   3778                             CD8VForm VForm> {
   3779 let ExeDomain = SSEPackedSingle in {
   3780   // Intrinsic operation, reg.
   3781   // Vector intrinsic operation, reg
   3782   def PSr : AVX512AIi8<opcps, MRMSrcReg,
   3783                     (outs RC:$dst), (ins RC:$src1, i32i8imm:$src2),
   3784                     !strconcat(OpcodeStr,
   3785                     "ps\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   3786                     [(set RC:$dst, (V4F32Int RC:$src1, imm:$src2))]>;
   3787 
   3788   // Vector intrinsic operation, mem
   3789   def PSm : AVX512AIi8<opcps, MRMSrcMem,
   3790                     (outs RC:$dst), (ins x86memop:$src1, i32i8imm:$src2),
   3791                     !strconcat(OpcodeStr,
   3792                     "ps\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   3793                     [(set RC:$dst,
   3794                           (V4F32Int (mem_frag32 addr:$src1),imm:$src2))]>,
   3795                     EVEX_CD8<32, VForm>;
   3796 } // ExeDomain = SSEPackedSingle
   3797 
   3798 let ExeDomain = SSEPackedDouble in {
   3799   // Vector intrinsic operation, reg
   3800   def PDr : AVX512AIi8<opcpd, MRMSrcReg,
   3801                      (outs RC:$dst), (ins RC:$src1, i32i8imm:$src2),
   3802                      !strconcat(OpcodeStr,
   3803                      "pd\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   3804                      [(set RC:$dst, (V2F64Int RC:$src1, imm:$src2))]>;
   3805 
   3806   // Vector intrinsic operation, mem
   3807   def PDm : AVX512AIi8<opcpd, MRMSrcMem,
   3808                      (outs RC:$dst), (ins x86memop:$src1, i32i8imm:$src2),
   3809                      !strconcat(OpcodeStr,
   3810                      "pd\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   3811                      [(set RC:$dst,
   3812                           (V2F64Int (mem_frag64 addr:$src1),imm:$src2))]>,
   3813                      EVEX_CD8<64, VForm>;
   3814 } // ExeDomain = SSEPackedDouble
   3815 }
   3816 
   3817 multiclass avx512_fp_binop_rm<bits<8> opcss, bits<8> opcsd,
   3818                             string OpcodeStr,
   3819                             Intrinsic F32Int,
   3820                             Intrinsic F64Int> {
   3821 let ExeDomain = GenericDomain in {
   3822   // Operation, reg.
   3823   let hasSideEffects = 0 in
   3824   def SSr : AVX512AIi8<opcss, MRMSrcReg,
   3825       (outs FR32X:$dst), (ins FR32X:$src1, FR32X:$src2, i32i8imm:$src3),
   3826       !strconcat(OpcodeStr,
   3827               "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
   3828       []>;
   3829 
   3830   // Intrinsic operation, reg.
   3831   let isCodeGenOnly = 1 in
   3832   def SSr_Int : AVX512AIi8<opcss, MRMSrcReg,
   3833         (outs VR128X:$dst), (ins VR128X:$src1, VR128X:$src2, i32i8imm:$src3),
   3834         !strconcat(OpcodeStr,
   3835                 "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
   3836         [(set VR128X:$dst, (F32Int VR128X:$src1, VR128X:$src2, imm:$src3))]>;
   3837 
   3838   // Intrinsic operation, mem.
   3839   def SSm : AVX512AIi8<opcss, MRMSrcMem, (outs VR128X:$dst),
   3840                      (ins VR128X:$src1, ssmem:$src2, i32i8imm:$src3),
   3841                      !strconcat(OpcodeStr,
   3842                    "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
   3843                      [(set VR128X:$dst, (F32Int VR128X:$src1, 
   3844                                          sse_load_f32:$src2, imm:$src3))]>,
   3845                      EVEX_CD8<32, CD8VT1>;
   3846 
   3847   // Operation, reg.
   3848   let hasSideEffects = 0 in
   3849   def SDr : AVX512AIi8<opcsd, MRMSrcReg,
   3850         (outs FR64X:$dst), (ins FR64X:$src1, FR64X:$src2, i32i8imm:$src3),
   3851         !strconcat(OpcodeStr,
   3852                 "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
   3853         []>, VEX_W;
   3854 
   3855   // Intrinsic operation, reg.
   3856   let isCodeGenOnly = 1 in
   3857   def SDr_Int : AVX512AIi8<opcsd, MRMSrcReg,
   3858         (outs VR128X:$dst), (ins VR128X:$src1, VR128X:$src2, i32i8imm:$src3),
   3859         !strconcat(OpcodeStr,
   3860                 "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
   3861         [(set VR128X:$dst, (F64Int VR128X:$src1, VR128X:$src2, imm:$src3))]>,
   3862         VEX_W;
   3863 
   3864   // Intrinsic operation, mem.
   3865   def SDm : AVX512AIi8<opcsd, MRMSrcMem,
   3866         (outs VR128X:$dst), (ins VR128X:$src1, sdmem:$src2, i32i8imm:$src3),
   3867         !strconcat(OpcodeStr,
   3868                 "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
   3869         [(set VR128X:$dst,
   3870               (F64Int VR128X:$src1, sse_load_f64:$src2, imm:$src3))]>,
   3871         VEX_W, EVEX_CD8<64, CD8VT1>;
   3872 } // ExeDomain = GenericDomain
   3873 }
   3874 
   3875 multiclass avx512_rndscale<bits<8> opc, string OpcodeStr,
   3876                             X86MemOperand x86memop, RegisterClass RC,
   3877                             PatFrag mem_frag, Domain d> {
   3878 let ExeDomain = d in {
   3879   // Intrinsic operation, reg.
   3880   // Vector intrinsic operation, reg
   3881   def r : AVX512AIi8<opc, MRMSrcReg,
   3882                     (outs RC:$dst), (ins RC:$src1, i32i8imm:$src2),
   3883                     !strconcat(OpcodeStr,
   3884                     " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   3885                     []>, EVEX;
   3886 
   3887   // Vector intrinsic operation, mem
   3888   def m : AVX512AIi8<opc, MRMSrcMem,
   3889                     (outs RC:$dst), (ins x86memop:$src1, i32i8imm:$src2),
   3890                     !strconcat(OpcodeStr,
   3891                     " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   3892                     []>, EVEX;
   3893 } // ExeDomain
   3894 }
   3895 
   3896 
   3897 defm VRNDSCALEPSZ : avx512_rndscale<0x08, "vrndscaleps", f512mem, VR512,
   3898                                 memopv16f32, SSEPackedSingle>, EVEX_V512,
   3899                                 EVEX_CD8<32, CD8VF>;
   3900 
   3901 def : Pat<(v16f32 (int_x86_avx512_mask_rndscale_ps_512 (v16f32 VR512:$src1),
   3902                    imm:$src2, (v16f32 VR512:$src1), (i16 -1),
   3903                    FROUND_CURRENT)),
   3904                    (VRNDSCALEPSZr VR512:$src1, imm:$src2)>;
   3905 
   3906 
   3907 defm VRNDSCALEPDZ : avx512_rndscale<0x09, "vrndscalepd", f512mem, VR512,
   3908                                 memopv8f64, SSEPackedDouble>, EVEX_V512,
   3909                                 VEX_W, EVEX_CD8<64, CD8VF>;
   3910 
   3911 def : Pat<(v8f64 (int_x86_avx512_mask_rndscale_pd_512 (v8f64 VR512:$src1),
   3912                   imm:$src2, (v8f64 VR512:$src1), (i8 -1),
   3913                   FROUND_CURRENT)),
   3914                    (VRNDSCALEPDZr VR512:$src1, imm:$src2)>;
   3915 
   3916 multiclass avx512_rndscale_scalar<bits<8> opc, string OpcodeStr,
   3917                      Operand x86memop, RegisterClass RC, Domain d> {
   3918 let ExeDomain = d in {
   3919   def r : AVX512AIi8<opc, MRMSrcReg,
   3920                     (outs RC:$dst), (ins RC:$src1, RC:$src2, i32i8imm:$src3),
   3921                     !strconcat(OpcodeStr,
   3922                     " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   3923                     []>, EVEX_4V;
   3924 
   3925   def m : AVX512AIi8<opc, MRMSrcMem,
   3926                     (outs RC:$dst), (ins RC:$src1, x86memop:$src2,  i32i8imm:$src3),
   3927                     !strconcat(OpcodeStr,
   3928                     " \t{$src2, $src1, $dst|$dst, $src1, $src2}"),
   3929                     []>, EVEX_4V;
   3930 } // ExeDomain
   3931 }
   3932 
   3933 defm VRNDSCALESS : avx512_rndscale_scalar<0x0A, "vrndscaless", ssmem, FR32X,
   3934                                 SSEPackedSingle>, EVEX_CD8<32, CD8VT1>;
   3935                                 
   3936 defm VRNDSCALESD : avx512_rndscale_scalar<0x0B, "vrndscalesd", sdmem, FR64X,
   3937                                 SSEPackedDouble>, EVEX_CD8<64, CD8VT1>;
   3938 
   3939 def : Pat<(ffloor FR32X:$src),
   3940           (VRNDSCALESSr (f32 (IMPLICIT_DEF)), FR32X:$src, (i32 0x1))>;
   3941 def : Pat<(f64 (ffloor FR64X:$src)),
   3942           (VRNDSCALESDr (f64 (IMPLICIT_DEF)), FR64X:$src, (i32 0x1))>;
   3943 def : Pat<(f32 (fnearbyint FR32X:$src)),
   3944           (VRNDSCALESSr (f32 (IMPLICIT_DEF)), FR32X:$src, (i32 0xC))>;
   3945 def : Pat<(f64 (fnearbyint FR64X:$src)),
   3946           (VRNDSCALESDr (f64 (IMPLICIT_DEF)), FR64X:$src, (i32 0xC))>;
   3947 def : Pat<(f32 (fceil FR32X:$src)),
   3948           (VRNDSCALESSr (f32 (IMPLICIT_DEF)), FR32X:$src, (i32 0x2))>;
   3949 def : Pat<(f64 (fceil FR64X:$src)),
   3950           (VRNDSCALESDr (f64 (IMPLICIT_DEF)), FR64X:$src, (i32 0x2))>;
   3951 def : Pat<(f32 (frint FR32X:$src)),
   3952           (VRNDSCALESSr (f32 (IMPLICIT_DEF)), FR32X:$src, (i32 0x4))>;
   3953 def : Pat<(f64 (frint FR64X:$src)),
   3954           (VRNDSCALESDr (f64 (IMPLICIT_DEF)), FR64X:$src, (i32 0x4))>;
   3955 def : Pat<(f32 (ftrunc FR32X:$src)),
   3956           (VRNDSCALESSr (f32 (IMPLICIT_DEF)), FR32X:$src, (i32 0x3))>;
   3957 def : Pat<(f64 (ftrunc FR64X:$src)),
   3958           (VRNDSCALESDr (f64 (IMPLICIT_DEF)), FR64X:$src, (i32 0x3))>;
   3959 
   3960 def : Pat<(v16f32 (ffloor VR512:$src)),
   3961           (VRNDSCALEPSZr VR512:$src, (i32 0x1))>;
   3962 def : Pat<(v16f32 (fnearbyint VR512:$src)),
   3963           (VRNDSCALEPSZr VR512:$src, (i32 0xC))>;
   3964 def : Pat<(v16f32 (fceil VR512:$src)),
   3965           (VRNDSCALEPSZr VR512:$src, (i32 0x2))>;
   3966 def : Pat<(v16f32 (frint VR512:$src)),
   3967           (VRNDSCALEPSZr VR512:$src, (i32 0x4))>;
   3968 def : Pat<(v16f32 (ftrunc VR512:$src)),
   3969           (VRNDSCALEPSZr VR512:$src, (i32 0x3))>;
   3970 
   3971 def : Pat<(v8f64 (ffloor VR512:$src)),
   3972           (VRNDSCALEPDZr VR512:$src, (i32 0x1))>;
   3973 def : Pat<(v8f64 (fnearbyint VR512:$src)),
   3974           (VRNDSCALEPDZr VR512:$src, (i32 0xC))>;
   3975 def : Pat<(v8f64 (fceil VR512:$src)),
   3976           (VRNDSCALEPDZr VR512:$src, (i32 0x2))>;
   3977 def : Pat<(v8f64 (frint VR512:$src)),
   3978           (VRNDSCALEPDZr VR512:$src, (i32 0x4))>;
   3979 def : Pat<(v8f64 (ftrunc VR512:$src)),
   3980           (VRNDSCALEPDZr VR512:$src, (i32 0x3))>;
   3981 
   3982 //-------------------------------------------------
   3983 // Integer truncate and extend operations
   3984 //-------------------------------------------------
   3985 
   3986 multiclass avx512_trunc_sat<bits<8> opc, string OpcodeStr,
   3987                           RegisterClass dstRC, RegisterClass srcRC,
   3988                           RegisterClass KRC, X86MemOperand x86memop> {
   3989   def rr : AVX512XS8I<opc, MRMDestReg, (outs dstRC:$dst),
   3990                (ins srcRC:$src),
   3991                !strconcat(OpcodeStr," \t{$src, $dst|$dst, $src}"),
   3992                []>, EVEX;
   3993 
   3994   def rrk : AVX512XS8I<opc, MRMDestReg, (outs dstRC:$dst),
   3995                (ins KRC:$mask, srcRC:$src),
   3996                !strconcat(OpcodeStr,
   3997                  " \t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}"),
   3998                []>, EVEX, EVEX_K;
   3999 
   4000   def rrkz : AVX512XS8I<opc, MRMDestReg, (outs dstRC:$dst),
   4001                (ins KRC:$mask, srcRC:$src),
   4002                !strconcat(OpcodeStr,
   4003                  " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"),
   4004                []>, EVEX, EVEX_KZ;
   4005 
   4006   def mr : AVX512XS8I<opc, MRMDestMem, (outs), (ins x86memop:$dst, srcRC:$src),
   4007                !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
   4008                []>, EVEX;
   4009 
   4010   def mrk : AVX512XS8I<opc, MRMDestMem, (outs),
   4011                (ins x86memop:$dst, KRC:$mask, srcRC:$src),
   4012                !strconcat(OpcodeStr, " \t{$src, $dst {${mask}}|${dst} {${mask}}, $src}"),
   4013                []>, EVEX, EVEX_K;
   4014 
   4015 }
   4016 defm VPMOVQB    : avx512_trunc_sat<0x32, "vpmovqb",   VR128X, VR512, VK8WM, 
   4017                                  i128mem>, EVEX_V512, EVEX_CD8<8, CD8VO>;
   4018 defm VPMOVSQB   : avx512_trunc_sat<0x22, "vpmovsqb",  VR128X, VR512, VK8WM,
   4019                                  i128mem>, EVEX_V512, EVEX_CD8<8, CD8VO>;
   4020 defm VPMOVUSQB  : avx512_trunc_sat<0x12, "vpmovusqb", VR128X, VR512, VK8WM,
   4021                                  i128mem>, EVEX_V512, EVEX_CD8<8, CD8VO>;
   4022 defm VPMOVQW    : avx512_trunc_sat<0x34, "vpmovqw",   VR128X, VR512, VK8WM,
   4023                                  i128mem>, EVEX_V512, EVEX_CD8<16, CD8VQ>;
   4024 defm VPMOVSQW   : avx512_trunc_sat<0x24, "vpmovsqw",  VR128X, VR512, VK8WM,
   4025                                  i128mem>, EVEX_V512, EVEX_CD8<16, CD8VQ>;
   4026 defm VPMOVUSQW  : avx512_trunc_sat<0x14, "vpmovusqw", VR128X, VR512, VK8WM,
   4027                                  i128mem>, EVEX_V512, EVEX_CD8<16, CD8VQ>;
   4028 defm VPMOVQD    : avx512_trunc_sat<0x35, "vpmovqd",   VR256X, VR512, VK8WM,
   4029                                  i256mem>, EVEX_V512, EVEX_CD8<32, CD8VH>;
   4030 defm VPMOVSQD   : avx512_trunc_sat<0x25, "vpmovsqd",  VR256X, VR512, VK8WM,
   4031                                  i256mem>, EVEX_V512, EVEX_CD8<32, CD8VH>;
   4032 defm VPMOVUSQD  : avx512_trunc_sat<0x15, "vpmovusqd", VR256X, VR512, VK8WM,
   4033                                  i256mem>, EVEX_V512, EVEX_CD8<32, CD8VH>;
   4034 defm VPMOVDW    : avx512_trunc_sat<0x33, "vpmovdw",   VR256X, VR512, VK16WM,
   4035                                  i256mem>, EVEX_V512, EVEX_CD8<16, CD8VH>;
   4036 defm VPMOVSDW   : avx512_trunc_sat<0x23, "vpmovsdw",  VR256X, VR512, VK16WM,
   4037                                  i256mem>, EVEX_V512, EVEX_CD8<16, CD8VH>;
   4038 defm VPMOVUSDW  : avx512_trunc_sat<0x13, "vpmovusdw", VR256X, VR512, VK16WM,
   4039                                  i256mem>, EVEX_V512, EVEX_CD8<16, CD8VH>;
   4040 defm VPMOVDB    : avx512_trunc_sat<0x31, "vpmovdb",   VR128X, VR512, VK16WM,
   4041                                  i128mem>, EVEX_V512, EVEX_CD8<8, CD8VQ>;
   4042 defm VPMOVSDB   : avx512_trunc_sat<0x21, "vpmovsdb",  VR128X, VR512, VK16WM,
   4043                                  i128mem>, EVEX_V512, EVEX_CD8<8, CD8VQ>;
   4044 defm VPMOVUSDB  : avx512_trunc_sat<0x11, "vpmovusdb", VR128X, VR512, VK16WM,
   4045                                  i128mem>, EVEX_V512, EVEX_CD8<8, CD8VQ>;
   4046 
   4047 def : Pat<(v16i8  (X86vtrunc (v8i64  VR512:$src))), (VPMOVQBrr  VR512:$src)>;
   4048 def : Pat<(v8i16  (X86vtrunc (v8i64  VR512:$src))), (VPMOVQWrr  VR512:$src)>;
   4049 def : Pat<(v16i16 (X86vtrunc (v16i32 VR512:$src))), (VPMOVDWrr  VR512:$src)>;
   4050 def : Pat<(v16i8  (X86vtrunc (v16i32 VR512:$src))), (VPMOVDBrr  VR512:$src)>;
   4051 def : Pat<(v8i32  (X86vtrunc (v8i64  VR512:$src))), (VPMOVQDrr  VR512:$src)>;
   4052 
   4053 def : Pat<(v16i8  (X86vtruncm VK16WM:$mask, (v16i32 VR512:$src))),
   4054                   (VPMOVDBrrkz VK16WM:$mask, VR512:$src)>;
   4055 def : Pat<(v16i16 (X86vtruncm VK16WM:$mask, (v16i32 VR512:$src))),
   4056                   (VPMOVDWrrkz VK16WM:$mask, VR512:$src)>;
   4057 def : Pat<(v8i16  (X86vtruncm VK8WM:$mask,  (v8i64 VR512:$src))),
   4058                   (VPMOVQWrrkz  VK8WM:$mask, VR512:$src)>;
   4059 def : Pat<(v8i32  (X86vtruncm VK8WM:$mask,  (v8i64 VR512:$src))),
   4060                   (VPMOVQDrrkz  VK8WM:$mask, VR512:$src)>;
   4061 
   4062 
   4063 multiclass avx512_extend<bits<8> opc, string OpcodeStr, RegisterClass KRC,
   4064                       RegisterClass DstRC, RegisterClass SrcRC, SDNode OpNode,
   4065                       PatFrag mem_frag, X86MemOperand x86memop,
   4066                       ValueType OpVT, ValueType InVT> {
   4067 
   4068   def rr : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst),
   4069               (ins SrcRC:$src),
   4070               !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
   4071               [(set DstRC:$dst, (OpVT (OpNode (InVT SrcRC:$src))))]>, EVEX;
   4072 
   4073   def rrk : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst),
   4074               (ins KRC:$mask, SrcRC:$src),
   4075               !strconcat(OpcodeStr, " \t{$src, $dst {${mask}} |$dst {${mask}}, $src}"),
   4076               []>, EVEX, EVEX_K;
   4077 
   4078   def rrkz : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst),
   4079               (ins KRC:$mask, SrcRC:$src),
   4080               !strconcat(OpcodeStr, " \t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}"),
   4081               []>, EVEX, EVEX_KZ;
   4082 
   4083   let mayLoad = 1 in {
   4084     def rm : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst),
   4085               (ins x86memop:$src),
   4086               !strconcat(OpcodeStr," \t{$src, $dst|$dst, $src}"),
   4087               [(set DstRC:$dst,
   4088                 (OpVT (OpNode (InVT (bitconvert (mem_frag addr:$src))))))]>,
   4089               EVEX;
   4090 
   4091     def rmk : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst),
   4092               (ins KRC:$mask, x86memop:$src),
   4093               !strconcat(OpcodeStr," \t{$src, $dst {${mask}} |$dst {${mask}}, $src}"),
   4094               []>,
   4095               EVEX, EVEX_K;
   4096 
   4097     def rmkz : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst),
   4098               (ins KRC:$mask, x86memop:$src),
   4099               !strconcat(OpcodeStr," \t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}"),
   4100               []>,
   4101               EVEX, EVEX_KZ;
   4102   }
   4103 }
   4104 
   4105 defm VPMOVZXBDZ: avx512_extend<0x31, "vpmovzxbd", VK16WM, VR512, VR128X, X86vzext,
   4106                              memopv2i64, i128mem, v16i32, v16i8>, EVEX_V512,
   4107                              EVEX_CD8<8, CD8VQ>;
   4108 defm VPMOVZXBQZ: avx512_extend<0x32, "vpmovzxbq", VK8WM, VR512, VR128X, X86vzext,
   4109                              memopv2i64, i128mem, v8i64, v16i8>, EVEX_V512,
   4110                              EVEX_CD8<8, CD8VO>;
   4111 defm VPMOVZXWDZ: avx512_extend<0x33, "vpmovzxwd", VK16WM, VR512, VR256X, X86vzext,
   4112                              memopv4i64, i256mem, v16i32, v16i16>, EVEX_V512,
   4113                              EVEX_CD8<16, CD8VH>;
   4114 defm VPMOVZXWQZ: avx512_extend<0x34, "vpmovzxwq", VK8WM, VR512, VR128X, X86vzext,
   4115                              memopv2i64, i128mem, v8i64, v8i16>, EVEX_V512,
   4116                              EVEX_CD8<16, CD8VQ>;
   4117 defm VPMOVZXDQZ: avx512_extend<0x35, "vpmovzxdq", VK8WM, VR512, VR256X, X86vzext,
   4118                              memopv4i64, i256mem, v8i64, v8i32>, EVEX_V512,
   4119                              EVEX_CD8<32, CD8VH>;
   4120 
   4121 defm VPMOVSXBDZ: avx512_extend<0x21, "vpmovsxbd", VK16WM, VR512, VR128X, X86vsext,
   4122                              memopv2i64, i128mem, v16i32, v16i8>, EVEX_V512,
   4123                              EVEX_CD8<8, CD8VQ>;
   4124 defm VPMOVSXBQZ: avx512_extend<0x22, "vpmovsxbq", VK8WM, VR512, VR128X, X86vsext,
   4125                              memopv2i64, i128mem, v8i64, v16i8>, EVEX_V512,
   4126                              EVEX_CD8<8, CD8VO>;
   4127 defm VPMOVSXWDZ: avx512_extend<0x23, "vpmovsxwd", VK16WM, VR512, VR256X, X86vsext,
   4128                              memopv4i64, i256mem, v16i32, v16i16>, EVEX_V512,
   4129                              EVEX_CD8<16, CD8VH>;
   4130 defm VPMOVSXWQZ: avx512_extend<0x24, "vpmovsxwq", VK8WM, VR512, VR128X, X86vsext,
   4131                              memopv2i64, i128mem, v8i64, v8i16>, EVEX_V512,
   4132                              EVEX_CD8<16, CD8VQ>;
   4133 defm VPMOVSXDQZ: avx512_extend<0x25, "vpmovsxdq", VK8WM, VR512, VR256X, X86vsext,
   4134                              memopv4i64, i256mem, v8i64, v8i32>, EVEX_V512,
   4135                              EVEX_CD8<32, CD8VH>;
   4136 
   4137 //===----------------------------------------------------------------------===//
   4138 // GATHER - SCATTER Operations
   4139 
   4140 multiclass avx512_gather<bits<8> opc, string OpcodeStr, RegisterClass KRC,
   4141                        RegisterClass RC, X86MemOperand memop> {
   4142 let mayLoad = 1,
   4143   Constraints = "@earlyclobber $dst, $src1 = $dst, $mask = $mask_wb" in
   4144   def rm  : AVX5128I<opc, MRMSrcMem, (outs RC:$dst, KRC:$mask_wb),
   4145             (ins RC:$src1, KRC:$mask, memop:$src2),
   4146             !strconcat(OpcodeStr,
   4147             " \t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"),
   4148             []>, EVEX, EVEX_K;
   4149 }
   4150 
   4151 let ExeDomain = SSEPackedDouble in {
   4152 defm VGATHERDPDZ : avx512_gather<0x92, "vgatherdpd", VK8WM, VR512, vy64xmem>,
   4153                                  EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
   4154 defm VGATHERQPDZ : avx512_gather<0x93, "vgatherqpd", VK8WM, VR512, vz64mem>,
   4155                                  EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
   4156 }
   4157 
   4158 let ExeDomain = SSEPackedSingle in {
   4159 defm VGATHERDPSZ : avx512_gather<0x92, "vgatherdps", VK16WM, VR512, vz32mem>,
   4160                                  EVEX_V512, EVEX_CD8<32, CD8VT1>;
   4161 defm VGATHERQPSZ : avx512_gather<0x93, "vgatherqps", VK8WM, VR256X, vz64mem>,
   4162                                  EVEX_V512, EVEX_CD8<32, CD8VT1>;
   4163 }
   4164   
   4165 defm VPGATHERDQZ : avx512_gather<0x90, "vpgatherdq", VK8WM, VR512,  vy64xmem>,
   4166                                  EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
   4167 defm VPGATHERDDZ : avx512_gather<0x90, "vpgatherdd", VK16WM, VR512, vz32mem>,
   4168                                  EVEX_V512, EVEX_CD8<32, CD8VT1>;
   4169 
   4170 defm VPGATHERQQZ : avx512_gather<0x91, "vpgatherqq", VK8WM, VR512,  vz64mem>,
   4171                                  EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
   4172 defm VPGATHERQDZ : avx512_gather<0x91, "vpgatherqd", VK8WM, VR256X,  vz64mem>,
   4173                                  EVEX_V512, EVEX_CD8<32, CD8VT1>;
   4174 
   4175 multiclass avx512_scatter<bits<8> opc, string OpcodeStr, RegisterClass KRC,
   4176                        RegisterClass RC, X86MemOperand memop> {
   4177 let mayStore = 1, Constraints = "$mask = $mask_wb" in
   4178   def mr  : AVX5128I<opc, MRMDestMem, (outs KRC:$mask_wb),
   4179             (ins memop:$dst, KRC:$mask, RC:$src2),
   4180             !strconcat(OpcodeStr,
   4181             " \t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"),
   4182             []>, EVEX, EVEX_K;
   4183 }
   4184 
   4185 let ExeDomain = SSEPackedDouble in {
   4186 defm VSCATTERDPDZ : avx512_scatter<0xA2, "vscatterdpd", VK8WM, VR512, vy64xmem>,
   4187                                    EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
   4188 defm VSCATTERQPDZ : avx512_scatter<0xA3, "vscatterqpd", VK8WM, VR512, vz64mem>,
   4189                                    EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
   4190 }
   4191 
   4192 let ExeDomain = SSEPackedSingle in {
   4193 defm VSCATTERDPSZ : avx512_scatter<0xA2, "vscatterdps", VK16WM, VR512, vz32mem>,
   4194                                    EVEX_V512, EVEX_CD8<32, CD8VT1>;
   4195 defm VSCATTERQPSZ : avx512_scatter<0xA3, "vscatterqps", VK8WM, VR256X, vz64mem>,
   4196                                    EVEX_V512, EVEX_CD8<32, CD8VT1>;
   4197 }
   4198 
   4199 defm VPSCATTERDQZ : avx512_scatter<0xA0, "vpscatterdq", VK8WM, VR512, vy64xmem>,
   4200                                    EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
   4201 defm VPSCATTERDDZ : avx512_scatter<0xA0, "vpscatterdd", VK16WM, VR512, vz32mem>,
   4202                                    EVEX_V512, EVEX_CD8<32, CD8VT1>;
   4203 
   4204 defm VPSCATTERQQZ : avx512_scatter<0xA1, "vpscatterqq", VK8WM, VR512, vz64mem>,
   4205                                   EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
   4206 defm VPSCATTERQDZ : avx512_scatter<0xA1, "vpscatterqd", VK8WM, VR256X, vz64mem>,
   4207                                   EVEX_V512, EVEX_CD8<32, CD8VT1>;
   4208 
   4209 // prefetch
   4210 multiclass avx512_gather_scatter_prefetch<bits<8> opc, Format F, string OpcodeStr,
   4211                        RegisterClass KRC, X86MemOperand memop> {
   4212   let Predicates = [HasPFI], hasSideEffects = 1 in
   4213   def m  : AVX5128I<opc, F, (outs), (ins KRC:$mask, memop:$src),
   4214             !strconcat(OpcodeStr, " \t{$src {${mask}}|{${mask}}, $src}"),
   4215             []>, EVEX, EVEX_K;
   4216 }
   4217 
   4218 defm VGATHERPF0DPS: avx512_gather_scatter_prefetch<0xC6, MRM1m, "vgatherpf0dps",
   4219                      VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;
   4220 
   4221 defm VGATHERPF0QPS: avx512_gather_scatter_prefetch<0xC7, MRM1m, "vgatherpf0qps",
   4222                      VK8WM, vz64mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>;
   4223 
   4224 defm VGATHERPF0DPD: avx512_gather_scatter_prefetch<0xC6, MRM1m, "vgatherpf0dpd",
   4225                      VK8WM, vy32mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>;
   4226 
   4227 defm VGATHERPF0QPD: avx512_gather_scatter_prefetch<0xC7, MRM1m, "vgatherpf0qpd",
   4228                      VK8WM, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
   4229                      
   4230 defm VGATHERPF1DPS: avx512_gather_scatter_prefetch<0xC6, MRM2m, "vgatherpf1dps",
   4231                      VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;
   4232 
   4233 defm VGATHERPF1QPS: avx512_gather_scatter_prefetch<0xC7, MRM2m, "vgatherpf1qps",
   4234                      VK8WM, vz64mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>;
   4235 
   4236 defm VGATHERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM2m, "vgatherpf1dpd",
   4237                      VK8WM, vy32mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>;
   4238 
   4239 defm VGATHERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM2m, "vgatherpf1qpd",
   4240                      VK8WM, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
   4241 
   4242 defm VSCATTERPF0DPS: avx512_gather_scatter_prefetch<0xC6, MRM5m, "vscatterpf0dps",
   4243                      VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;
   4244 
   4245 defm VSCATTERPF0QPS: avx512_gather_scatter_prefetch<0xC7, MRM5m, "vscatterpf0qps",
   4246                      VK8WM, vz64mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>;
   4247 
   4248 defm VSCATTERPF0DPD: avx512_gather_scatter_prefetch<0xC6, MRM5m, "vscatterpf0dpd",
   4249                      VK8WM, vy32mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>;
   4250 
   4251 defm VSCATTERPF0QPD: avx512_gather_scatter_prefetch<0xC7, MRM5m, "vscatterpf0qpd",
   4252                      VK8WM, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
   4253 
   4254 defm VSCATTERPF1DPS: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dps",
   4255                      VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;
   4256 
   4257 defm VSCATTERPF1QPS: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qps",
   4258                      VK8WM, vz64mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>;
   4259 
   4260 defm VSCATTERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dpd",
   4261                      VK8WM, vy32mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>;
   4262 
   4263 defm VSCATTERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qpd",
   4264                      VK8WM, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
   4265 //===----------------------------------------------------------------------===//
   4266 // VSHUFPS - VSHUFPD Operations
   4267 
   4268 multiclass avx512_shufp<RegisterClass RC, X86MemOperand x86memop,
   4269                       ValueType vt, string OpcodeStr, PatFrag mem_frag,
   4270                       Domain d> {
   4271   def rmi : AVX512PIi8<0xC6, MRMSrcMem, (outs RC:$dst),
   4272                    (ins RC:$src1, x86memop:$src2, i8imm:$src3),
   4273                    !strconcat(OpcodeStr,
   4274                    " \t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
   4275                    [(set RC:$dst, (vt (X86Shufp RC:$src1, (mem_frag addr:$src2),
   4276                                        (i8 imm:$src3))))], d, IIC_SSE_SHUFP>,
   4277                    EVEX_4V, Sched<[WriteShuffleLd, ReadAfterLd]>;
   4278   def rri : AVX512PIi8<0xC6, MRMSrcReg, (outs RC:$dst),
   4279                    (ins RC:$src1, RC:$src2, i8imm:$src3),
   4280                    !strconcat(OpcodeStr,
   4281                    " \t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
   4282                    [(set RC:$dst, (vt (X86Shufp RC:$src1, RC:$src2,
   4283                                        (i8 imm:$src3))))], d, IIC_SSE_SHUFP>,
   4284                    EVEX_4V, Sched<[WriteShuffle]>;
   4285 }
   4286 
   4287 defm VSHUFPSZ  : avx512_shufp<VR512, f512mem, v16f32, "vshufps", memopv16f32,
   4288                   SSEPackedSingle>, PS, EVEX_V512, EVEX_CD8<32, CD8VF>;
   4289 defm VSHUFPDZ  : avx512_shufp<VR512, f512mem, v8f64, "vshufpd", memopv8f64,
   4290                   SSEPackedDouble>, PD, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
   4291 
   4292 def : Pat<(v16i32 (X86Shufp VR512:$src1, VR512:$src2, (i8 imm:$imm))),
   4293           (VSHUFPSZrri VR512:$src1, VR512:$src2, imm:$imm)>;
   4294 def : Pat<(v16i32 (X86Shufp VR512:$src1,
   4295                     (memopv16i32 addr:$src2), (i8 imm:$imm))),
   4296           (VSHUFPSZrmi VR512:$src1, addr:$src2, imm:$imm)>;
   4297 
   4298 def : Pat<(v8i64 (X86Shufp VR512:$src1, VR512:$src2, (i8 imm:$imm))),
   4299           (VSHUFPDZrri VR512:$src1, VR512:$src2, imm:$imm)>;
   4300 def : Pat<(v8i64 (X86Shufp VR512:$src1,
   4301                             (memopv8i64 addr:$src2), (i8 imm:$imm))),
   4302           (VSHUFPDZrmi VR512:$src1, addr:$src2, imm:$imm)>;
   4303 
   4304 multiclass avx512_alignr<string OpcodeStr, RegisterClass RC,
   4305                        X86MemOperand x86memop> {
   4306   def rri : AVX512AIi8<0x03, MRMSrcReg, (outs RC:$dst),
   4307                      (ins RC:$src1, RC:$src2, i8imm:$src3),
   4308                      !strconcat(OpcodeStr,
   4309                      " \t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
   4310                      []>, EVEX_4V;
   4311   let mayLoad = 1 in
   4312   def rmi : AVX512AIi8<0x03, MRMSrcMem, (outs RC:$dst),
   4313                      (ins RC:$src1, x86memop:$src2, i8imm:$src3),
   4314                      !strconcat(OpcodeStr,
   4315                      " \t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
   4316                      []>, EVEX_4V;
   4317 }
   4318 defm VALIGND : avx512_alignr<"valignd", VR512, i512mem>, 
   4319                  EVEX_V512, EVEX_CD8<32, CD8VF>;
   4320 defm VALIGNQ : avx512_alignr<"valignq", VR512, i512mem>, 
   4321                  VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
   4322 
   4323 def : Pat<(v16f32 (X86PAlignr VR512:$src1, VR512:$src2, (i8 imm:$imm))),
   4324           (VALIGNDrri VR512:$src2, VR512:$src1, imm:$imm)>;
   4325 def : Pat<(v8f64 (X86PAlignr VR512:$src1, VR512:$src2, (i8 imm:$imm))),
   4326           (VALIGNQrri VR512:$src2, VR512:$src1, imm:$imm)>;
   4327 def : Pat<(v16i32 (X86PAlignr VR512:$src1, VR512:$src2, (i8 imm:$imm))),
   4328           (VALIGNDrri VR512:$src2, VR512:$src1, imm:$imm)>;
   4329 def : Pat<(v8i64 (X86PAlignr VR512:$src1, VR512:$src2, (i8 imm:$imm))),
   4330           (VALIGNQrri VR512:$src2, VR512:$src1, imm:$imm)>;
   4331 
   4332 // Helper fragments to match sext vXi1 to vXiY.
   4333 def v16i1sextv16i32  : PatLeaf<(v16i32 (X86vsrai VR512:$src, (i8 31)))>;
   4334 def v8i1sextv8i64  : PatLeaf<(v8i64 (X86vsrai VR512:$src, (i8 63)))>;
   4335 
   4336 multiclass avx512_vpabs<bits<8> opc, string OpcodeStr, ValueType OpVT,
   4337                         RegisterClass KRC, RegisterClass RC,
   4338                         X86MemOperand x86memop, X86MemOperand x86scalar_mop,
   4339                         string BrdcstStr> {
   4340   def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src),
   4341             !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
   4342             []>, EVEX;
   4343   def rrk : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins KRC:$mask, RC:$src),
   4344              !strconcat(OpcodeStr, " \t{$src, $dst {${mask}}|$dst {${mask}}, $src}"),
   4345              []>, EVEX, EVEX_K;
   4346   def rrkz : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), (ins KRC:$mask, RC:$src),
   4347               !strconcat(OpcodeStr,
   4348                          " \t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}"),
   4349               []>, EVEX, EVEX_KZ;
   4350   let mayLoad = 1 in {
   4351     def rm : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst),
   4352               (ins x86memop:$src),
   4353               !strconcat(OpcodeStr, " \t{$src, $dst|$dst, $src}"),
   4354               []>, EVEX;
   4355     def rmk : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst),
   4356                (ins KRC:$mask, x86memop:$src),
   4357                !strconcat(OpcodeStr,
   4358                           " \t{$src, $dst {${mask}}|$dst {${mask}}, $src}"),
   4359                []>, EVEX, EVEX_K;
   4360     def rmkz : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst),
   4361                 (ins KRC:$mask, x86memop:$src),
   4362                 !strconcat(OpcodeStr,
   4363                            " \t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}"),
   4364                 []>, EVEX, EVEX_KZ;
   4365     def rmb : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst),
   4366                (ins x86scalar_mop:$src),
   4367                !strconcat(OpcodeStr, " \t{${src}", BrdcstStr,
   4368                           ", $dst|$dst, ${src}", BrdcstStr, "}"),
   4369                []>, EVEX, EVEX_B;
   4370     def rmbk : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst),
   4371                 (ins KRC:$mask, x86scalar_mop:$src),
   4372                 !strconcat(OpcodeStr, " \t{${src}", BrdcstStr,
   4373                            ", $dst {${mask}}|$dst {${mask}}, ${src}", BrdcstStr, "}"),
   4374                 []>, EVEX, EVEX_B, EVEX_K;
   4375     def rmbkz : AVX5128I<opc, MRMSrcMem, (outs VR512:$dst),
   4376                  (ins KRC:$mask, x86scalar_mop:$src),
   4377                  !strconcat(OpcodeStr, " \t{${src}", BrdcstStr,
   4378                             ", $dst {${mask}} {z}|$dst {${mask}} {z}, ${src}",
   4379                             BrdcstStr, "}"),
   4380                  []>, EVEX, EVEX_B, EVEX_KZ;
   4381   }
   4382 }
   4383 
   4384 defm VPABSDZ : avx512_vpabs<0x1E, "vpabsd", v16i32, VK16WM, VR512,
   4385                            i512mem, i32mem, "{1to16}">, EVEX_V512,
   4386                            EVEX_CD8<32, CD8VF>;
   4387 defm VPABSQZ : avx512_vpabs<0x1F, "vpabsq", v8i64, VK8WM, VR512,
   4388                            i512mem, i64mem, "{1to8}">, EVEX_V512, VEX_W,
   4389                            EVEX_CD8<64, CD8VF>;
   4390 
   4391 def : Pat<(xor
   4392           (bc_v16i32 (v16i1sextv16i32)),
   4393           (bc_v16i32 (add (v16i32 VR512:$src), (v16i1sextv16i32)))),
   4394           (VPABSDZrr VR512:$src)>;
   4395 def : Pat<(xor
   4396           (bc_v8i64 (v8i1sextv8i64)),
   4397           (bc_v8i64 (add (v8i64 VR512:$src), (v8i1sextv8i64)))),
   4398           (VPABSQZrr VR512:$src)>;
   4399 
   4400 def : Pat<(v16i32 (int_x86_avx512_mask_pabs_d_512 (v16i32 VR512:$src),
   4401                    (v16i32 immAllZerosV), (i16 -1))),
   4402           (VPABSDZrr VR512:$src)>;
   4403 def : Pat<(v8i64 (int_x86_avx512_mask_pabs_q_512 (v8i64 VR512:$src),
   4404                    (bc_v8i64 (v16i32 immAllZerosV)), (i8 -1))),
   4405           (VPABSQZrr VR512:$src)>;
   4406 
   4407 multiclass avx512_conflict<bits<8> opc, string OpcodeStr, 
   4408                         RegisterClass RC, RegisterClass KRC,
   4409                         X86MemOperand x86memop,
   4410                         X86MemOperand x86scalar_mop, string BrdcstStr> {
   4411   def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst),
   4412        (ins RC:$src),
   4413        !strconcat(OpcodeStr, " \t{$src, ${dst} |${dst}, $src}"),
   4414        []>, EVEX;
   4415   def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
   4416        (ins x86memop:$src),
   4417        !strconcat(OpcodeStr, " \t{$src, ${dst}|${dst}, $src}"),
   4418        []>, EVEX;
   4419   def rmb : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
   4420        (ins x86scalar_mop:$src),
   4421        !strconcat(OpcodeStr, " \t{${src}", BrdcstStr,
   4422                   ", ${dst}|${dst}, ${src}", BrdcstStr, "}"),
   4423        []>, EVEX, EVEX_B;
   4424   def rrkz : AVX5128I<opc, MRMSrcReg, (outs RC:$dst),
   4425        (ins KRC:$mask, RC:$src),
   4426        !strconcat(OpcodeStr,
   4427                   " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"),
   4428        []>, EVEX, EVEX_KZ;
   4429   def rmkz : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
   4430        (ins KRC:$mask, x86memop:$src),
   4431        !strconcat(OpcodeStr,
   4432                   " \t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"),
   4433        []>, EVEX, EVEX_KZ;
   4434   def rmbkz : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
   4435        (ins KRC:$mask, x86scalar_mop:$src),
   4436        !strconcat(OpcodeStr, " \t{${src}", BrdcstStr,
   4437                   ", ${dst} {${mask}} {z}|${dst} {${mask}} {z}, ${src}",
   4438                   BrdcstStr, "}"),
   4439        []>, EVEX, EVEX_KZ, EVEX_B;
   4440        
   4441   let Constraints = "$src1 = $dst" in {
   4442   def rrk : AVX5128I<opc, MRMSrcReg, (outs RC:$dst),
   4443        (ins RC:$src1, KRC:$mask, RC:$src2),
   4444        !strconcat(OpcodeStr,
   4445                   " \t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"),
   4446        []>, EVEX, EVEX_K;
   4447   def rmk : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
   4448        (ins RC:$src1, KRC:$mask, x86memop:$src2),
   4449        !strconcat(OpcodeStr,
   4450                   " \t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"),
   4451        []>, EVEX, EVEX_K;
   4452   def rmbk : AVX5128I<opc, MRMSrcMem, (outs RC:$dst),
   4453        (ins RC:$src1, KRC:$mask, x86scalar_mop:$src2),
   4454        !strconcat(OpcodeStr, " \t{${src2}", BrdcstStr,
   4455                   ", ${dst} {${mask}}|${dst} {${mask}}, ${src2}", BrdcstStr, "}"),
   4456        []>, EVEX, EVEX_K, EVEX_B;
   4457    }
   4458 }
   4459 
   4460 let Predicates = [HasCDI] in {
   4461 defm VPCONFLICTD : avx512_conflict<0xC4, "vpconflictd", VR512, VK16WM,
   4462                     i512mem, i32mem, "{1to16}">,
   4463                     EVEX_V512, EVEX_CD8<32, CD8VF>;
   4464 
   4465 
   4466 defm VPCONFLICTQ : avx512_conflict<0xC4, "vpconflictq", VR512, VK8WM,
   4467                     i512mem, i64mem, "{1to8}">,
   4468                     EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
   4469 
   4470 }
   4471 
   4472 def : Pat<(int_x86_avx512_mask_conflict_d_512 VR512:$src2, VR512:$src1,
   4473                                               GR16:$mask),
   4474           (VPCONFLICTDrrk VR512:$src1,
   4475            (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), VR512:$src2)>;
   4476 
   4477 def : Pat<(int_x86_avx512_mask_conflict_q_512 VR512:$src2, VR512:$src1,
   4478                                               GR8:$mask),
   4479           (VPCONFLICTQrrk VR512:$src1,
   4480            (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), VR512:$src2)>;
   4481 
   4482 let Predicates = [HasCDI] in {
   4483 defm VPLZCNTD : avx512_conflict<0x44, "vplzcntd", VR512, VK16WM,
   4484                     i512mem, i32mem, "{1to16}">,
   4485                     EVEX_V512, EVEX_CD8<32, CD8VF>;
   4486 
   4487 
   4488 defm VPLZCNTQ : avx512_conflict<0x44, "vplzcntq", VR512, VK8WM,
   4489                     i512mem, i64mem, "{1to8}">,
   4490                     EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
   4491 
   4492 }
   4493 
   4494 def : Pat<(int_x86_avx512_mask_lzcnt_d_512 VR512:$src2, VR512:$src1,
   4495                                               GR16:$mask),
   4496           (VPLZCNTDrrk VR512:$src1,
   4497            (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), VR512:$src2)>;
   4498 
   4499 def : Pat<(int_x86_avx512_mask_lzcnt_q_512 VR512:$src2, VR512:$src1,
   4500                                               GR8:$mask),
   4501           (VPLZCNTQrrk VR512:$src1,
   4502            (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), VR512:$src2)>;
   4503 
   4504 def : Pat<(v16i32 (ctlz (memopv16i32 addr:$src))),
   4505           (VPLZCNTDrm addr:$src)>;
   4506 def : Pat<(v16i32 (ctlz (v16i32 VR512:$src))),
   4507           (VPLZCNTDrr VR512:$src)>;
   4508 def : Pat<(v8i64 (ctlz (memopv8i64 addr:$src))),
   4509           (VPLZCNTQrm addr:$src)>;
   4510 def : Pat<(v8i64 (ctlz (v8i64 VR512:$src))),
   4511           (VPLZCNTQrr VR512:$src)>;
   4512 
   4513 def : Pat<(store (i1 -1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>;
   4514 def : Pat<(store (i1  1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>;
   4515 def : Pat<(store (i1  0), addr:$dst), (MOV8mi addr:$dst, (i8 0))>;
   4516 
   4517 def : Pat<(store VK1:$src, addr:$dst),
   4518           (KMOVWmk addr:$dst, (COPY_TO_REGCLASS VK1:$src, VK16))>;
   4519 
   4520 def truncstorei1 : PatFrag<(ops node:$val, node:$ptr),
   4521                            (truncstore node:$val, node:$ptr), [{
   4522   return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i1;
   4523 }]>;
   4524 
   4525 def : Pat<(truncstorei1 GR8:$src, addr:$dst),
   4526           (MOV8mr addr:$dst, GR8:$src)>;
   4527 
   4528