Lines Matching refs:VT
441 for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE;
442 VT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++VT) {
444 setOperationAction(ISD::SIGN_EXTEND_INREG, (MVT::SimpleValueType)VT,
447 setOperationAction(ISD::MULHS, (MVT::SimpleValueType)VT, Expand);
448 setOperationAction(ISD::SMUL_LOHI, (MVT::SimpleValueType)VT, Expand);
449 setOperationAction(ISD::MULHU, (MVT::SimpleValueType)VT, Expand);
450 setOperationAction(ISD::UMUL_LOHI, (MVT::SimpleValueType)VT, Expand);
452 setOperationAction(ISD::BSWAP, (MVT::SimpleValueType)VT, Expand);
456 setTruncStoreAction((MVT::SimpleValueType)VT,
458 setLoadExtAction(ISD::SEXTLOAD, (MVT::SimpleValueType)VT, Expand);
459 setLoadExtAction(ISD::ZEXTLOAD, (MVT::SimpleValueType)VT, Expand);
460 setLoadExtAction(ISD::EXTLOAD, (MVT::SimpleValueType)VT, Expand);
477 void AArch64TargetLowering::addTypeForNEON(EVT VT, EVT PromotedBitwiseVT) {
478 if (VT == MVT::v2f32) {
479 setOperationAction(ISD::LOAD, VT.getSimpleVT(), Promote);
480 AddPromotedToType(ISD::LOAD, VT.getSimpleVT(), MVT::v2i32);
482 setOperationAction(ISD::STORE, VT.getSimpleVT(), Promote);
483 AddPromotedToType(ISD::STORE, VT.getSimpleVT(), MVT::v2i32);
484 } else if (VT == MVT::v2f64 || VT == MVT::v4f32) {
485 setOperationAction(ISD::LOAD, VT.getSimpleVT(), Promote);
486 AddPromotedToType(ISD::LOAD, VT.getSimpleVT(), MVT::v2i64);
488 setOperationAction(ISD::STORE, VT.getSimpleVT(), Promote);
489 AddPromotedToType(ISD::STORE, VT.getSimpleVT(), MVT::v2i64);
493 if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) {
494 setOperationAction(ISD::FSIN, VT.getSimpleVT(), Expand);
495 setOperationAction(ISD::FCOS, VT.getSimpleVT(), Expand);
496 setOperationAction(ISD::FPOWI, VT.getSimpleVT(), Expand);
497 setOperationAction(ISD::FPOW, VT.getSimpleVT(), Expand);
498 setOperationAction(ISD::FLOG, VT.getSimpleVT(), Expand);
499 setOperationAction(ISD::FLOG2, VT.getSimpleVT(), Expand);
500 setOperationAction(ISD::FLOG10, VT.getSimpleVT(), Expand);
501 setOperationAction(ISD::FEXP, VT.getSimpleVT(), Expand);
502 setOperationAction(ISD::FEXP2, VT.getSimpleVT(), Expand);
505 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT.getSimpleVT(), Custom);
506 setOperationAction(ISD::INSERT_VECTOR_ELT, VT.getSimpleVT(), Custom);
507 setOperationAction(ISD::BUILD_VECTOR, VT.getSimpleVT(), Custom);
508 setOperationAction(ISD::VECTOR_SHUFFLE, VT.getSimpleVT(), Custom);
509 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT.getSimpleVT(), Custom);
510 setOperationAction(ISD::SRA, VT.getSimpleVT(), Custom);
511 setOperationAction(ISD::SRL, VT.getSimpleVT(), Custom);
512 setOperationAction(ISD::SHL, VT.getSimpleVT(), Custom);
513 setOperationAction(ISD::AND, VT.getSimpleVT(), Custom);
514 setOperationAction(ISD::OR, VT.getSimpleVT(), Custom);
515 setOperationAction(ISD::SETCC, VT.getSimpleVT(), Custom);
516 setOperationAction(ISD::CONCAT_VECTORS, VT.getSimpleVT(), Legal);
518 setOperationAction(ISD::SELECT, VT.getSimpleVT(), Expand);
519 setOperationAction(ISD::SELECT_CC, VT.getSimpleVT(), Expand);
520 setOperationAction(ISD::VSELECT, VT.getSimpleVT(), Expand);
521 setLoadExtAction(ISD::EXTLOAD, VT.getSimpleVT(), Expand);
524 if (VT != MVT::v8i8 && VT != MVT::v16i8)
525 setOperationAction(ISD::CTPOP, VT.getSimpleVT(), Expand);
527 setOperationAction(ISD::UDIV, VT.getSimpleVT(), Expand);
528 setOperationAction(ISD::SDIV, VT.getSimpleVT(), Expand);
529 setOperationAction(ISD::UREM, VT.getSimpleVT(), Expand);
530 setOperationAction(ISD::SREM, VT.getSimpleVT(), Expand);
531 setOperationAction(ISD::FREM, VT.getSimpleVT(), Expand);
533 setOperationAction(ISD::FP_TO_SINT, VT.getSimpleVT(), Custom);
534 setOperationAction(ISD::FP_TO_UINT, VT.getSimpleVT(), Custom);
539 setIndexedLoadAction(im, VT.getSimpleVT(), Legal);
540 setIndexedStoreAction(im, VT.getSimpleVT(), Legal);
545 void AArch64TargetLowering::addDRTypeForNEON(MVT VT) {
546 addRegisterClass(VT, &AArch64::FPR64RegClass);
547 addTypeForNEON(VT, MVT::v2i32);
550 void AArch64TargetLowering::addQRTypeForNEON(MVT VT) {
551 addRegisterClass(VT, &AArch64::FPR128RegClass);
552 addTypeForNEON(VT, MVT::v4i32);
555 EVT AArch64TargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const {
556 if (!VT.isVector())
558 return VT.changeVectorElementTypeToInteger();
586 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
587 unsigned MemBits = VT.getScalarType().getSizeInBits();
606 MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
608 if (VT == MVT::v8i8 || VT == MVT::v16i8) {
612 } else if (VT == MVT::v4i16 || VT == MVT::v8i16) {
975 EVT VT = LHS.getValueType();
977 if (VT.isFloatingPoint())
978 return DAG.getNode(AArch64ISD::FCMP, dl, VT, LHS, RHS);
1012 return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS)
1019 EVT VT = RHS.getValueType();
1028 if ((VT == MVT::i32 && C != 0x80000000 &&
1030 (VT == MVT::i64 && C != 0x80000000ULL &&
1033 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
1034 RHS = DAG.getConstant(C, VT);
1039 if ((VT == MVT::i32 && C != 0 &&
1041 (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) {
1043 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
1044 RHS = DAG.getConstant(C, VT);
1049 if ((VT == MVT::i32 && C != 0x7fffffff &&
1051 (VT == MVT::i64 && C != 0x7ffffffffffffffULL &&
1054 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
1055 RHS = DAG.getConstant(C, VT);
1060 if ((VT == MVT::i32 && C != 0xffffffff &&
1062 (VT == MVT::i64 && C != 0xfffffffffffffffULL &&
1065 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
1066 RHS = DAG.getConstant(C, VT);
1262 EVT VT = Op.getValueType();
1265 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
1268 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
1386 EVT VT = Op.getValueType();
1388 if (VT.getSizeInBits() < InVT.getSizeInBits()) {
1393 return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv);
1396 if (VT.getSizeInBits() > InVT.getSizeInBits()) {
1399 return DAG.getNode(Op.getOpcode(), dl, VT, Ext);
1434 EVT VT = Op.getValueType();
1439 if (VT.getSizeInBits() < InVT.getSizeInBits()) {
1444 return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0));
1447 if (VT.getSizeInBits() > InVT.getSizeInBits()) {
1450 EVT CastVT = VT.changeVectorElementTypeToInteger();
1452 return DAG.getNode(Op.getOpcode(), dl, VT, In);
1652 // At this point, Ins[].VT
1655 // Since AnalyzeFormalArguments uses Ins[].VT for both ValVT and LocVT, here
1662 MVT ValVT = Ins[i].VT;
1743 assert(RegVT == Ins[i].VT && "incorrect register location selected");
2157 MVT ArgVT = Outs[i].VT;
2166 // At this point, Outs[].VT may already be promoted to i32. To correctly
2169 // Since AnalyzeCallOperands uses Ins[].VT for both ValVT and LocVT, here
2174 MVT ValVT = Outs[i].VT;
2276 if (realArgIdx == 0 && Flags.isReturned() && Outs[0].VT == MVT::i64) {
2279 assert(!Ins.empty() && Ins[0].VT == MVT::i64 &&
2954 EVT VT = Op.getValueType();
2960 if (SrcVT != VT) {
2961 if (SrcVT == MVT::f32 && VT == MVT::f64)
2962 In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2);
2963 else if (SrcVT == MVT::f64 && VT == MVT::f32)
2964 In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0));
2966 // FIXME: Src type is different, bail out for now. Can VT really be a
2974 if (VT == MVT::f32 || VT == MVT::v2f32 || VT == MVT::v4f32) {
2979 if (!VT.isVector()) {
2988 } else if (VT == MVT::f64 || VT == MVT::v2f64) {
2997 if (!VT.isVector()) {
3018 if (VT == MVT::f64 || VT == MVT::v2f64) {
3027 if (VT == MVT::f32)
3028 return DAG.getTargetExtractSubreg(AArch64::ssub, DL, VT, Sel);
3029 else if (VT == MVT::f64)
3030 return DAG.getTargetExtractSubreg(AArch64::dsub, DL, VT, Sel);
3032 return DAG.getNode(ISD::BITCAST, DL, VT, Sel);
3050 EVT VT = Op.getValueType();
3054 if (VT == MVT::i32) {
3067 if (VT == MVT::i64)
3083 EVT VT = Op.getValueType();
3084 SDValue TVal = DAG.getConstant(1, VT);
3085 SDValue FVal = DAG.getConstant(0, VT);
3108 return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp);
3127 return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CC1Val, Cmp);
3137 DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
3140 return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
3319 EVT VT = Op.getValueType();
3320 return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp);
3326 EVT VT = Op.getValueType();
3351 return DAG.getNode(AArch64ISD::FMAX, dl, VT, MinMaxLHS, MinMaxRHS);
3359 return DAG.getNode(AArch64ISD::FMIN, dl, VT, MinMaxLHS, MinMaxRHS);
3374 SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
3380 return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
3584 EVT VT = Op.getValueType();
3602 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
3609 if (VT.isInteger() && !VT.isVector())
3612 if (VT.isFloatingPoint() && !VT.isVector() && VT != MVT::f64) {
3630 SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0),
3637 return DAG.getLoad(VT, DL, APStore, VAList, MachinePointerInfo(), false,
3646 EVT VT = Op.getValueType();
3650 DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, VT);
3652 FrameAddr = DAG.getLoad(VT, DL, DAG.getEntryNode(), FrameAddr,
3660 EVT VT) const {
3675 EVT VT = Op.getValueType();
3681 return DAG.getLoad(VT, DL, DAG.getEntryNode(),
3682 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
3688 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT);
3696 EVT VT = Op.getValueType();
3697 unsigned VTBits = VT.getSizeInBits();
3709 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
3712 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
3718 SDValue FalseValLo = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
3719 SDValue TrueValLo = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
3721 DAG.getNode(AArch64ISD::CSEL, dl, VT, TrueValLo, FalseValLo, CCVal, Cmp);
3725 SDValue FalseValHi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
3727 ? DAG.getNode(Opc, dl, VT, ShOpHi,
3729 : DAG.getConstant(0, VT);
3731 DAG.getNode(AArch64ISD::CSEL, dl, VT, TrueValHi, FalseValHi, CCVal, Cmp);
3742 EVT VT = Op.getValueType();
3743 unsigned VTBits = VT.getSizeInBits();
3753 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
3756 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
3757 SDValue Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
3759 SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
3765 DAG.getNode(AArch64ISD::CSEL, dl, VT, Tmp3, FalseVal, CCVal, Cmp);
3769 SDValue TrueValLo = DAG.getConstant(0, VT);
3770 SDValue FalseValLo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
3772 DAG.getNode(AArch64ISD::CSEL, dl, VT, TrueValLo, FalseValLo, CCVal, Cmp);
3784 bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
3787 if (Imm.isPosZero() && (VT == MVT::f64 || VT == MVT::f32))
3790 if (VT == MVT::f64)
3792 else if (VT == MVT::f32)
3883 const std::string &Constraint, MVT VT) const {
3887 if (VT.getSizeInBits() == 64)
3891 if (VT == MVT::f32)
3893 if (VT.getSizeInBits() == 64)
3895 if (VT.getSizeInBits() == 128)
3901 if (VT.getSizeInBits() == 128)
3912 Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT);
4078 EVT VT = V64Reg.getValueType();
4079 unsigned NarrowSize = VT.getVectorNumElements();
4080 MVT EltTy = VT.getVectorElementType().getSimpleVT();
4098 EVT VT = V128Reg.getValueType();
4099 unsigned WideSize = VT.getVectorNumElements();
4100 MVT EltTy = VT.getVectorElementType().getSimpleVT();
4113 EVT VT = Op.getValueType();
4114 unsigned NumElts = VT.getVectorNumElements();
4158 SDValue ShuffleSrcs[2] = { DAG.getUNDEF(VT), DAG.getUNDEF(VT) };
4168 VT.getVectorElementType()) {
4174 EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
4176 VT.getVectorElementType().getSizeInBits());
4185 if (CurSource.getValueType() == VT) {
4193 ShuffleSrcs[i] = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, CurSource,
4211 ShuffleSrcs[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, CurSource,
4216 ShuffleSrcs[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, CurSource,
4221 SDValue VEXTSrc1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, CurSource,
4223 SDValue VEXTSrc2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, CurSource,
4226 ShuffleSrcs[i] = DAG.getNode(AArch64ISD::EXT, dl, VT, VEXTSrc1, VEXTSrc2,
4252 if (isShuffleMaskLegal(Mask, VT))
4253 return DAG.getVectorShuffle(VT, dl, ShuffleSrcs[0], ShuffleSrcs[1],
4261 static bool isSingletonEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
4262 unsigned NumElts = VT.getVectorNumElements();
4292 static bool isEXTMask(ArrayRef<int> M, EVT VT, bool &ReverseEXT,
4299 unsigned NumElts = VT.getVectorNumElements();
4334 static bool isREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) {
4338 unsigned EltSz = VT.getVectorElementType().getSizeInBits();
4342 unsigned NumElts = VT.getVectorNumElements();
4361 static bool isZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
4362 unsigned NumElts = VT.getVectorNumElements();
4375 static bool isUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
4376 unsigned NumElts = VT.getVectorNumElements();
4388 static bool isTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
4389 unsigned NumElts = VT.getVectorNumElements();
4402 static bool isZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
4403 unsigned NumElts = VT.getVectorNumElements();
4419 static bool isUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
4420 unsigned Half = VT.getVectorNumElements() / 2;
4438 static bool isTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
4439 unsigned NumElts = VT.getVectorNumElements();
4488 static bool isConcatMask(ArrayRef<int> Mask, EVT VT, bool SplitLHS) {
4489 if (VT.getSizeInBits() != 128)
4492 unsigned NumElts = VT.getVectorNumElements();
4510 EVT VT = Op.getValueType();
4515 if (VT.getVectorElementType() != V0.getValueType().getVectorElementType() ||
4516 VT.getVectorElementType() != V1.getValueType().getVectorElementType())
4521 if (!isConcatMask(Mask, VT, SplitV0))
4524 EVT CastVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
4525 VT.getVectorNumElements() / 2);
4534 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, V0, V1);
4574 EVT VT = OpLHS.getValueType();
4581 if (VT.getVectorElementType() == MVT::i32 ||
4582 VT.getVectorElementType() == MVT::f32)
4583 return DAG.getNode(AArch64ISD::REV64, dl, VT, OpLHS);
4585 if (VT.getVectorElementType() == MVT::i16)
4586 return DAG.getNode(AArch64ISD::REV32, dl, VT, OpLHS);
4588 assert(VT.getVectorElementType() == MVT::i8);
4589 return DAG.getNode(AArch64ISD::REV16, dl, VT, OpLHS);
4594 EVT EltTy = VT.getVectorElementType();
4607 if (VT.getSizeInBits() == 64)
4610 return DAG.getNode(Opcode, dl, VT, OpLHS, Lane);
4616 return DAG.getNode(AArch64ISD::EXT, dl, VT, OpLHS, OpRHS,
4620 return DAG.getNode(AArch64ISD::UZP1, dl, DAG.getVTList(VT, VT), OpLHS,
4623 return DAG.getNode(AArch64ISD::UZP2, dl, DAG.getVTList(VT, VT), OpLHS,
4626 return DAG.getNode(AArch64ISD::ZIP1, dl, DAG.getVTList(VT, VT), OpLHS,
4629 return DAG.getNode(AArch64ISD::ZIP2, dl, DAG.getVTList(VT, VT), OpLHS,
4632 return DAG.getNode(AArch64ISD::TRN1, dl, DAG.getVTList(VT, VT), OpLHS,
4635 return DAG.getNode(AArch64ISD::TRN2, dl, DAG.getVTList(VT, VT), OpLHS,
4718 EVT VT = Op.getValueType();
4745 return DAG.getNode(AArch64ISD::DUP, dl, VT, V1.getOperand(Lane));
4757 unsigned Idx = Lane >= (int)VT.getVectorNumElements() / 2;
4758 Lane -= Idx * VT.getVectorNumElements() / 2;
4760 } else if (VT.getSizeInBits() == 64)
4763 return DAG.getNode(Opcode, dl, VT, V1, DAG.getConstant(Lane, MVT::i64));
4766 if (isREVMask(ShuffleMask, VT, 64))
4768 if (isREVMask(ShuffleMask, VT, 32))
4770 if (isREVMask(ShuffleMask, VT, 16))
4775 if (isEXTMask(ShuffleMask, VT, ReverseEXT, Imm)) {
4782 isSingletonEXTMask(ShuffleMask, VT, Imm)) {
4789 if (isZIPMask(ShuffleMask, VT, WhichResult)) {
4793 if (isUZPMask(ShuffleMask, VT, WhichResult)) {
4797 if (isTRNMask(ShuffleMask, VT, WhichResult)) {
4802 if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
4806 if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
4810 if (isTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
4830 SrcLane -= VT.getVectorNumElements();
4834 EVT ScalarVT = VT.getVectorElementType();
4839 ISD::INSERT_VECTOR_ELT, dl, VT, DstVec,
4846 unsigned NumElts = VT.getVectorNumElements();
4871 EVT VT = BVN->getValueType(0);
4876 unsigned NumSplats = VT.getSizeInBits() / SplatBitSize;
4881 CnstBits |= SplatBits.zextOrTrunc(VT.getSizeInBits());
4882 UndefBits |= (SplatBits ^ SplatUndef).zextOrTrunc(VT.getSizeInBits());
4897 EVT VT = Op.getValueType();
4902 APInt CnstBits(VT.getSizeInBits(), 0);
4903 APInt UndefBits(VT.getSizeInBits(), 0);
4919 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
4923 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
4928 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
4932 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
4937 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
4941 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
4946 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
4950 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
4955 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
4959 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
4964 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
4968 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
4995 EVT VT = Bvec->getValueType(0);
4996 unsigned NumElts = VT.getVectorNumElements();
5023 EVT VT = N->getValueType(0);
5025 if (!VT.isVector())
5057 unsigned ElemSizeInBits = VT.getVectorElementType().getSizeInBits();
5070 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
5095 EVT VT = Op.getValueType();
5105 APInt CnstBits(VT.getSizeInBits(), 0);
5106 APInt UndefBits(VT.getSizeInBits(), 0);
5119 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
5123 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5128 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
5132 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5137 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
5141 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5146 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
5150 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5155 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
5159 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5164 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
5168 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5190 EVT VT = Op.getValueType();
5191 EVT EltTy= VT.getVectorElementType();
5197 for (unsigned I = 0, E = VT.getVectorNumElements(); I != E; ++I) {
5206 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
5212 EVT VT = Op.getValueType();
5216 APInt CnstBits(VT.getSizeInBits(), 0);
5217 APInt UndefBits(VT.getSizeInBits(), 0);
5232 if (VT.isInteger() && (CnstVal == 0 || CnstVal == ~0ULL))
5238 if (VT.getSizeInBits() == 128) {
5241 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5247 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5252 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
5256 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5261 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
5265 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5270 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
5274 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5279 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
5283 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5288 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
5292 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5297 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
5301 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5306 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
5310 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5315 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
5319 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5324 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v16i8 : MVT::v8i8;
5327 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5333 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4f32 : MVT::v2f32;
5336 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5340 VT.getSizeInBits() == 128) {
5344 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5351 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
5355 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5360 MVT MovTy = (VT
5364 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5369 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
5373 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5378 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
5382 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5387 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
5391 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5396 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
5400 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5405 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
5409 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5414 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
5418 return DAG.getNode(ISD::BITCAST, dl, VT, Mov);
5441 unsigned NumElts = VT.getVectorNumElements();
5473 return DAG.getUNDEF(VT);
5476 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value);
5483 Value.getValueType() != VT)
5484 return DAG.getNode(AArch64ISD::DUP, dl, VT, Value);
5494 unsigned Opcode = getDUPLANEOp(VT.getVectorElementType());
5495 return DAG.getNode(Opcode, dl, VT, Value, Lane);
5498 if (VT.getVectorElementType().isFloatingPoint()) {
5501 (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
5508 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
5517 SDValue Val = DAG.getNode(AArch64ISD::DUP, dl, VT, ConstantValue);
5523 // Note that type legalization likely mucked about with the VT of the
5525 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, V, LaneIdx);
5551 SDValue Vec = DAG.getUNDEF(VT);
5553 unsigned ElemSize = VT.getVectorElementType().getSizeInBits();
5562 DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, dl, VT, Vec, Op0,
5572 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx);
5589 EVT VT = Op.getOperand(0).getValueType();
5592 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
5593 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64)
5596 if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 &&
5597 VT != MVT::v1i64 && VT != MVT::v2f32)
5621 EVT VT = Op.getOperand(0).getValueType();
5624 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
5625 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64)
5628 if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 &&
5629 VT != MVT::v1i64 && VT != MVT::v2f32)
5649 EVT VT = Op.getOperand(0).getValueType();
5652 if (!VT.isVector())
5681 if (Size == 64 && Val * VT.getVectorElementType().getSizeInBits() == 64)
5688 EVT VT) const {
5689 if (VT.getVectorNumElements() == 4 &&
5690 (VT.is128BitVector() || VT.is64BitVector())) {
5713 return (ShuffleVectorSDNode::isSplatMask(&M[0], VT) || isREVMask(M, VT, 64) ||
5714 isREVMask(M, VT, 32) || isREVMask(M, VT, 16) ||
5715 isEXTMask(M, VT, DummyBool, DummyUnsigned) ||
5716 // isTBLMask(M, VT) || // FIXME: Port TBL support from ARM.
5717 isTRNMask(M, VT, DummyUnsigned) || isUZPMask(M, VT, DummyUnsigned) ||
5718 isZIPMask(M, VT, DummyUnsigned) ||
5719 isTRN_v_undef_Mask(M, VT, DummyUnsigned) ||
5720 isUZP_v_undef_Mask(M, VT, DummyUnsigned) ||
5721 isZIP_v_undef_Mask(M, VT, DummyUnsigned) ||
5722 isINSMask(M, VT.getVectorNumElements(), DummyBool, DummyInt) ||
5723 isConcatMask(M, VT, VT.getSizeInBits() == 128));
5749 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
5750 assert(VT.isVector() && "vector shift count is not a vector type");
5751 unsigned ElementBits = VT.getVectorElementType().getSizeInBits();
5763 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic,
5765 assert(VT.isVector() && "vector shift count is not a vector type");
5766 VT.getVectorElementType().getSizeInBits();
5776 EVT VT = Op.getValueType();
5782 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
5789 if (isVShiftLImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize)
5790 return DAG.getNode(AArch64ISD::VSHL, SDLoc(Op), VT, Op.getOperand(0),
5792 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
5798 if (isVShiftRImm(Op.getOperand(1), VT, false, false, Cnt) &&
5802 return DAG.getNode(Opc, SDLoc(Op), VT, Op.getOperand(0),
5812 SDValue NegShift = DAG.getNode(AArch64ISD::NEG, DL, VT, Op.getOperand(1));
5814 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
5823 AArch64CC::CondCode CC, bool NoNans, EVT VT,
5828 APInt CnstBits(VT.getSizeInBits(), 0);
5829 APInt UndefBits(VT.getSizeInBits(), 0);
5840 Fcmeq = DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS);
5842 Fcmeq = DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS);
5843 return DAG.getNode(AArch64ISD::NOT, dl, VT, Fcmeq);
5847 return DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS);
5848 return DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS);
5851 return DAG.getNode(AArch64ISD::FCMGEz, dl, VT, LHS);
5852 return DAG.getNode(AArch64ISD::FCMGE, dl, VT, LHS, RHS);
5855 return DAG.getNode(AArch64ISD::FCMGTz, dl, VT, LHS);
5856 return DAG.getNode(AArch64ISD::FCMGT, dl, VT, LHS, RHS);
5859 return DAG.getNode(AArch64ISD::FCMLEz, dl, VT, LHS);
5860 return DAG.getNode(AArch64ISD::FCMGE, dl, VT, RHS, LHS);
5868 return DAG.getNode(AArch64ISD::FCMLTz, dl, VT, LHS);
5869 return DAG.getNode(AArch64ISD::FCMGT, dl, VT, RHS, LHS);
5879 Cmeq = DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS);
5881 Cmeq = DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS);
5882 return DAG.getNode(AArch64ISD::NOT, dl, VT, Cmeq);
5886 return DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS);
5887 return DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS);
5890 return DAG.getNode(AArch64ISD::CMGEz, dl, VT, LHS);
5891 return DAG.getNode(AArch64ISD::CMGE, dl, VT, LHS, RHS);
5894 return DAG.getNode(AArch64ISD::CMGTz, dl, VT, LHS);
5895 return DAG.getNode(AArch64ISD::CMGT, dl, VT, LHS, RHS);
5898 return DAG.getNode(AArch64ISD::CMLEz, dl, VT, LHS);
5899 return DAG.getNode(AArch64ISD::CMGE, dl, VT, RHS, LHS);
5901 return DAG.getNode(AArch64ISD::CMHS, dl, VT, RHS, LHS);
5903 return DAG.getNode(AArch64ISD::CMHI, dl, VT, RHS, LHS);
5906 return DAG.getNode(AArch64ISD::CMLTz, dl, VT, LHS);
5907 return DAG.getNode(AArch64ISD::CMGT, dl, VT, RHS, LHS);
5909 return DAG.getNode(AArch64ISD::CMHI, dl, VT, LHS, RHS);
5911 return DAG.getNode(AArch64ISD::CMHS, dl, VT, LHS, RHS);
6253 bool AArch64TargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const {
6254 VT = VT.getScalarType();
6256 if (!VT.isSimple())
6259 switch (VT.getSimpleVT().SimpleTy) {
6283 EVT VT = N->getValueType(0);
6286 if (N->getOpcode() == ISD::AND && (VT == MVT::i32 || VT == MVT::i64) &&
6322 EVT VT = N->getValueType(0);
6330 if (VT.isInteger() && N->getOpcode() == ISD::XOR &&
6334 if (Y1C->getAPIntValue() == VT.getSizeInBits() - 1) {
6335 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT),
6339 DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::i32),
6340 N0.getOperand(0), DAG.getConstant(0, VT));
6341 return DAG.getNode(AArch64ISD::CSEL, DL, VT, N0.getOperand(0), Neg,
6371 EVT VT = N->getValueType(0);
6377 DAG.getNode(ISD::SHL, SDLoc(N), VT, N->getOperand(0),
6379 return DAG.getNode(ISD::ADD, SDLoc(N), VT, ShiftedVal,
6386 DAG.getNode(ISD::SHL, SDLoc(N), VT, N->getOperand(0),
6388 return DAG.getNode(ISD::SUB, SDLoc(N), VT, ShiftedVal,
6396 DAG.getNode(ISD::SHL, SDLoc(N), VT, N->getOperand(0),
6399 DAG.getNode(ISD::ADD, SDLoc(N), VT, ShiftedVal, N->getOperand(0));
6400 return DAG.getNode(ISD::SUB, SDLoc(N), VT, DAG.getConstant(0, VT), Add);
6406 DAG.getNode(ISD::SHL, SDLoc(N), VT, N->getOperand(0),
6408 return DAG.getNode(ISD::SUB, SDLoc(N), VT, N->getOperand(0),
6417 EVT VT = N->getValueType(0);
6418 if (VT != MVT::f32 && VT != MVT::f64)
6421 if (VT.getSizeInBits() != N->getOperand(0).getValueType().getSizeInBits())
6432 SDValue Load = DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(),
6443 return DAG.getNode(Opcode, SDLoc(N), VT, Load);
6477 EVT VT = N->getValueType(0);
6481 if (VT != MVT::i32 && VT != MVT::i64)
6501 if (ShiftLHS + ShiftRHS != VT.getSizeInBits())
6509 return DAG.getNode(AArch64ISD::EXTR, DL, VT, LHS, RHS,
6515 EVT VT = N->getValueType(0);
6519 if (!VT.isVector())
6532 unsigned Bits = VT.getVectorElementType().getSizeInBits();
6542 for (unsigned k = 0; k < VT.getVectorNumElements(); ++k) {
6553 return DAG.getNode(AArch64ISD::BSL, DL, VT, SDValue(BVN0, 0),
6566 EVT VT = N->getValueType(0);
6568 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
6598 EVT VT = N->getValueType(0);
6599 if (!VT.isVector())
6601 if (VT.getSimpleVT().getSizeInBits() != 64)
6628 if (SVT.getVectorNumElements() != VT.getVectorNumElements() * 2)
6636 unsigned NumElements = VT.getVectorNumElements();
6639 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Source, HalfIdx);
6642 return SDValue(DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, VT,
6657 EVT VT = N->getValueType(0);
6662 if (N->getOperand(0) == N->getOperand(1) && VT.getVectorNumElements() == 2) {
6663 assert(VT.getVectorElementType().getSizeInBits() == 64);
6664 return DAG.getNode(AArch64ISD::DUPLANE64, dl, VT,
6692 ISD::BITCAST, dl, VT,
6924 EVT VT = Op->getValueType(0);
6925 LHS = DAG.getNode(ISD::ADD, dl, VT, RHS, DAG.getConstant(1, VT));
6926 return DAG.getNode(AArch64ISD::CSEL, dl, VT, RHS, LHS, CCVal, Cmp);
6946 MVT VT = N->getSimpleValueType(0);
6947 if (!VT.is128BitVector()) {
6970 RHS = DAG.getNode(ExtType, SDLoc(N), VT, RHS);
6976 LHS = DAG.getNode(ExtType, SDLoc(N), VT, LHS);
6979 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, LHS, RHS);
7186 // If the vector type isn't a simple VT, it's beyond the scope of what
7194 // If the source VT is a 64-bit vector, we can play games and get the
7235 EVT VT = StVal.getValueType();
7239 if (VT.isFloatingPoint())
7247 unsigned NumVecElts = VT.getVectorNumElements();
7312 EVT VT = StVal.getValueType();
7316 if (!VT.isVector() || VT.getVectorNumElements() < 2 || VT == MVT::v2i64)
7324 if (VT.getSizeInBits() != 128 || S->getAlignment() >= 16 ||
7336 unsigned NumElts = VT.getVectorNumElements() / 2;
7337 // Split VT into two.
7339 EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), NumElts);
7364 EVT VT = N->getValueType(0);
7375 if (MemVT != VT.getVectorElementType())
7411 unsigned NumBytes = VT.getScalarSizeInBits() / 8;
7426 EVT Tys[3] = { VT, MVT::i64, MVT::Other };
7846 EVT VT;
7849 VT = LD->getMemoryVT();
7852 VT = ST->getMemoryVT();
7867 EVT VT;
7870 VT = LD->getMemoryVT();
7873 VT = ST->getMemoryVT();
7916 AArch64TargetLowering::getPreferredVectorAction(EVT VT) const {
7917 MVT SVT = VT.getSimpleVT();
7924 return TargetLoweringBase::getPreferredVectorAction(VT);
