Lines Matching refs:Ops
1741 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
1748 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift);
1750 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
1752 return Builder.CreateCall(F, Ops, name);
2474 SmallVectorImpl<Value *> &Ops,
2497 std::swap(Ops[0], Ops[1]);
2513 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
2517 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
2520 Ops[j] =
2521 CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType());
2522 Ops[j] =
2523 CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0);
2526 Value *Result = CGF.EmitNeonCall(F, Ops, s);
2538 SmallVectorImpl<llvm::Value *> &Ops, llvm::Value *Align) {
2564 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs");
2565 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs");
2571 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
2572 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
2573 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn");
2579 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn");
2582 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn");
2588 std::swap(Ops[0], Ops[1]);
2598 return EmitNeonCall(F, Ops, NameHint);
2604 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef()));
2608 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2610 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
2611 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
2625 return EmitNeonCall(F, Ops, "vcvt_n");
2643 return EmitNeonCall(F, Ops, "vcvt_n");
2659 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
2660 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt")
2661 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
2702 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
2706 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
2711 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2712 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2714 return Builder.CreateShuffleVector(Ops[0], Ops[1], SV, "vext");
2719 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2720 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2721 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
2724 return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]);
2728 Ops.push_back(Align);
2729 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vld1");
2737 Ops[1] = Builder.CreateCall2(F, Ops[1], Align, NameHint);
2738 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
2739 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2740 return Builder.CreateStore(Ops[1], Ops[0]);
2746 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2747 LoadInst *Ld = Builder.CreateLoad(Ops[0]);
2750 Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
2751 return EmitNeonSplat(Ops[0], CI);
2760 for (unsigned I = 2; I < Ops.size() - 1; ++I)
2761 Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
2762 Ops.push_back(Align);
2763 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint);
2764 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
2765 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2766 return Builder.CreateStore(Ops[1], Ops[0]);
2770 Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
2772 return Builder.CreateZExt(Ops[0], Ty, "vmovl");
2773 return Builder.CreateSExt(Ops[0], Ty, "vmovl");
2777 Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
2778 return Builder.CreateTrunc(Ops[0], Ty, "vmovn");
2788 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
2798 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
2808 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl");
2812 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end());
2817 AccumOps.push_back(Ops[0]);
2824 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n",
2831 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
2835 Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false);
2836 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
2840 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
2842 Ops[0] = Builder.CreateZExt(Ops[0], VTy);
2844 Ops[0] = Builder.CreateSExt(Ops[0], VTy);
2845 Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false);
2846 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n");
2850 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
2851 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false);
2853 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
2855 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
2856 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n");
2860 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n");
2875 Ops.push_back(Align);
2876 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "");
2882 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
2883 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
2884 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn");
2890 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn");
2893 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn");
2897 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
2898 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2899 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
2908 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
2910 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn");
2917 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2918 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2919 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
2920 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
2922 return Builder.CreateSExt(Ops[0], Ty, "vtst");
2926 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
2927 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2928 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
2936 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
2938 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp");
2945 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
2946 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2947 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
2956 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
2958 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip");
2970 Value *Result = EmitNeonCall(F, Ops, NameHint);
2998 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
3008 llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType());
3015 int PairPos = 0, End = Ops.size() - 1;
3017 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
3018 Ops[PairPos+1], SV, Name));
3026 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
3078 SmallVector<Value*, 2> Ops;
3080 Ops.push_back(EmitScalarExpr(E->getArg(i)));
3084 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
3239 SmallVector<Value*, 4> Ops;
3270 Ops.push_back(Src.first);
3296 Ops.push_back(Src.first);
3301 Ops.push_back(EmitScalarExpr(E->getArg(i)));
3318 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
3330 Ops.push_back(EmitScalarExpr(E->getArg(2)));
3331 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
3335 Ops.push_back(EmitScalarExpr(E->getArg(0)));
3336 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops,
3339 Ops.push_back(EmitScalarExpr(E->getArg(2)));
3340 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops,
3343 Ops.push_back(EmitScalarExpr(E->getArg(2)));
3344 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops,
3347 Ops.push_back(EmitScalarExpr(E->getArg(2)));
3348 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops,
3373 return Builder.CreateCall(F, Ops, "vcvtr");
3394 Builtin->NameHint, Builtin->TypeModifier, E, Ops, Align);
3404 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3405 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
3407 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
3411 Value *Ld = Builder.CreateCall2(F, Ops[0], Align);
3417 return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane");
3421 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3423 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3424 LoadInst *Ld = Builder.CreateLoad(Ops[0]);
3426 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane");
3446 Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld_dup");
3447 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
3448 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3449 return Builder.CreateStore(Ops[1], Ops[0]);
3467 Args.push_back(Ops[1]);
3474 Ops[1] = Builder.CreateCall(F, Args, "vld_dup");
3477 Value *Val = Builder.CreateExtractValue(Ops[1], i);
3481 Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i);
3483 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
3484 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3485 return Builder.CreateStore(Ops[1], Ops[0]);
3490 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n",
3494 Ops, "vqrshrun_n", 1, true);
3498 Ops, "vqshlu", 1, false);
3501 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n",
3505 Ops, "vqshrun_n", 1, true);
3509 Ops, "vrecpe");
3512 Ops, "vrshrn_n", 1, true);
3516 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", 1, true);
3519 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3520 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3521 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true);
3523 Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Ty), Ops[1], Ops[2]);
3524 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n");
3530 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift);
3532 Ops, "vsli_n");
3535 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3536 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
3537 return Builder.CreateAdd(Ops[0], Ops[1]);
3542 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3543 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
3544 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
3545 Ops[2] = Align;
3547 Ops[1]->getType()), Ops);
3551 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3552 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
3553 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
3554 StoreInst *St = Builder.CreateStore(Ops[1],
3555 Builder.CreateBitCast(Ops[0], Ty));
3561 Ops, "vtbl1");
3564 Ops, "vtbl2");
3567 Ops, "vtbl3");
3570 Ops, "vtbl4");
3573 Ops, "vtbx1");
3576 Ops, "vtbx2");
3579 Ops, "vtbx3");
3582 Ops, "vtbx4");
3588 SmallVectorImpl<Value *> &Ops) {
3647 TblOps.push_back(Ops[0]);
3648 return packTBLDVectorList(CGF, TblOps, nullptr, Ops[1], Ty,
3652 TblOps.push_back(Ops[0]);
3653 TblOps.push_back(Ops[1]);
3654 return packTBLDVectorList(CGF, TblOps, nullptr, Ops[2], Ty,
3658 TblOps.push_back(Ops[0]);
3659 TblOps.push_back(Ops[1]);
3660 TblOps.push_back(Ops[2]);
3661 return packTBLDVectorList(CGF, TblOps, nullptr, Ops[3], Ty,
3665 TblOps.push_back(Ops[0]);
3666 TblOps.push_back(Ops[1]);
3667 TblOps.push_back(Ops[2]);
3668 TblOps.push_back(Ops[3]);
3669 return packTBLDVectorList(CGF, TblOps, nullptr, Ops[4], Ty,
3673 TblOps.push_back(Ops[1]);
3674 Value *TblRes = packTBLDVectorList(CGF, TblOps, nullptr, Ops[2], Ty,
3679 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
3682 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
3687 TblOps.push_back(Ops[1]);
3688 TblOps.push_back(Ops[2]);
3689 return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[3], Ty,
3693 TblOps.push_back(Ops[1]);
3694 TblOps.push_back(Ops[2]);
3695 TblOps.push_back(Ops[3]);
3696 Value *TblRes = packTBLDVectorList(CGF, TblOps, nullptr, Ops[4], Ty,
3701 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
3705 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
3710 TblOps.push_back(Ops[1]);
3711 TblOps.push_back(Ops[2]);
3712 TblOps.push_back(Ops[3]);
3713 TblOps.push_back(Ops[4]);
3714 return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[5], Ty,
3748 return CGF.EmitNeonCall(F, Ops, s);
3770 emitVectorWrappedScalar8Intrinsic(unsigned Int, SmallVectorImpl<Value*> &Ops,
3777 Ops[0] = vectorWrapScalar8(Ops[0]);
3778 Ops[1] = vectorWrapScalar8(Ops[1]);
3780 Value *V = EmitNeonCall(CGM.getIntrinsic(Int, VTy), Ops, Name);
3786 emitVectorWrappedScalar16Intrinsic(unsigned Int, SmallVectorImpl<Value*> &Ops,
3793 Ops[0] = vectorWrapScalar16(Ops[0]);
3794 Ops[1] = vectorWrapScalar16(Ops[1]);
3796 Value *V = EmitNeonCall(CGM.getIntrinsic(Int, VTy), Ops, Name);
3821 SmallVector<Value*, 2> Ops;
3823 Ops.push_back(EmitScalarExpr(E->getArg(i)));
3827 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
3957 llvm::SmallVector<Value*, 4> Ops;
3959 Ops.push_back(EmitScalarExpr(E->getArg(i)));
3966 Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1)));
3967 Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E);
3992 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy);
4001 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4002 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
4005 Ops[0] = Builder.CreateBitCast(Ops[0], FTy);
4007 return Builder.CreateFPToUI(Ops[0], InTy);
4008 return Builder.CreateFPToSI(Ops[0], InTy);
4016 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4017 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
4020 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
4022 return Builder.CreateUIToFP(Ops[0], FTy);
4023 return Builder.CreateSIToFP(Ops[0], FTy);
4067 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4069 Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OEQ,
4074 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4076 Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGE,
4081 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4083 Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLE,
4088 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4090 Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGT,
4095 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4097 Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLT,
4102 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4103 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4104 Ops[0] = Builder.CreateICmp(llvm::ICmpInst::ICMP_EQ, Ops[0],
4106 return Builder.CreateSExt(Ops[0], Ty, "vceqzd");
4122 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4123 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
4124 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
4125 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
4126 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd");
4142 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4143 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
4144 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy);
4145 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
4146 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd");
4172 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4173 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
4174 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
4175 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]);
4176 Ops[0], Int64Ty, "vceqd");
4181 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4182 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4183 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4184 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
4185 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
4187 return Builder.CreateSExt(Ops[0], Ty, "vtstd");
4199 Ops.push_back(EmitScalarExpr(E->getArg(2)));
4200 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
4203 Ops[1] = Builder.CreateBitCast(Ops[1],
4205 Ops.push_back(EmitScalarExpr(E->getArg(2)));
4206 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
4209 Ops[1] = Builder.CreateBitCast(Ops[1],
4211 Ops.push_back(EmitScalarExpr(E->getArg(2)));
4212 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
4216 Ops[0] = Builder.CreateBitCast(Ops[0],
4218 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4222 Ops[0] = Builder.CreateBitCast(Ops[0],
4224 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4228 Ops[0] = Builder.CreateBitCast(Ops[0],
4230 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4234 Ops[0] = Builder.CreateBitCast(Ops[0],
4236 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4240 Ops[0] = Builder.CreateBitCast(
4241 Ops[0],
4243 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4246 Ops[0] = Builder.CreateBitCast(Ops[0],
4248 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4252 Ops[0] = Builder.CreateBitCast(Ops[0],
4254 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4258 Ops[0] = Builder.CreateBitCast(Ops[0],
4260 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4263 Ops[0] = Builder.CreateBitCast(Ops[0],
4265 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4269 Ops[0] = Builder.CreateBitCast(Ops[0],
4271 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4274 Ops[0] = Builder.CreateBitCast(Ops[0],
4276 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4279 Ops[0] = Builder.CreateBitCast(Ops[0],
4281 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4285 Ops[0] = Builder.CreateBitCast(Ops[0],
4287 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4291 Ops[0] = Builder.CreateBitCast(Ops[0],
4293 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4297 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd");
4300 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd");
4304 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
4307 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
4310 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
4315 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl");
4318 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4319 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
4321 Ops, "vqshlu_n");
4328 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4329 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
4330 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n");
4337 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4338 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
4339 Ops[1] = ConstantInt::get(Int64Ty, -SV);
4340 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n");
4347 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
4348 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2))));
4349 Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Int64Ty), Ops[1],
4350 Builder.CreateSExt(Ops[2], Int64Ty));
4351 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty));
4357 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n");
4362 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
4372 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt),
4377 Ops[1] = Builder.CreateAShr(
4378 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
4381 return Builder.CreateAdd(Ops[0], Ops[1]);
4387 // As Op + 0 = Op, return Ops[0] directly.
4389 return Ops[0];
4390 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt),
4392 return Builder.CreateAdd(Ops[0], Ops[1]);
4398 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
4401 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
4402 ProductOps.push_back(vectorWrapScalar16(Ops[2]));
4404 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
4407 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
4408 Ops.pop_back();
4414 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl");
4419 ProductOps.push_back(Ops[1]);
4421 Ops[1] =
4428 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl");
4434 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
4437 ProductOps.push_back(Ops[1]);
4438 ProductOps.push_back(Ops[2]);
4439 Ops[1] =
4442 Ops.pop_back();
4448 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl");
4465 Builtin->NameHint, Builtin->TypeModifier, E, Ops, nullptr);
4467 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops))
4476 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl");
4477 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl");
4478 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl");
4480 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl");
4481 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl");
4482 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl");
4483 return Builder.CreateBitCast(Ops[0], Ty);
4489 Value *Addend = Ops[0];
4490 Value *Multiplicand = Ops[1];
4491 Value *LaneSource = Ops[2];
4492 Ops[0] = Multiplicand;
4493 Ops[1] = LaneSource;
4494 Ops[2] = Addend;
4500 llvm::Constant *cst = cast<Constant>(Ops[3]);
4502 OpsOps[1], SourceTy);
4503 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane");
4505 Ops.pop_back();
4507 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla");
4513 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
4514 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
4517 Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
4518 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
4520 Value *Result = Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]);
4524 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4525 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4529 Ops[2] = Builder.CreateBitCast(Ops[2], STy);
4531 cast<ConstantInt>(Ops[3]));
4532 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
4534 return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]);
4538 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4539 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4541 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4542 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
4543 return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]);
4549 Ops.push_back(EmitScalarExpr(E->getArg(3)));
4552 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
4553 return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]);
4562 Value *Subtrahend = Ops[0];
4563 Value *Multiplicand = Ops[2];
4564 Ops[0] = Multiplicand;
4565 Ops[2] = Subtrahend;
4566 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
4567 Ops[1] = Builder.CreateFNeg(Ops[1]);
4569 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmls");
4575 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
4581 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax");
4587 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin");
4593 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd");
4604 TmpOps.push_back(Ops[1]);
4607 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
4615 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin");
4621 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax");
4625 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm");
4629 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm");
4632 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4634 Ops, "vrecps");
4638 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4640 Ops, "vrecps");
4648 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", 1, true);
4655 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", 1, false);
4659 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n");
4663 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n");
4667 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n");
4671 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n");
4675 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n");
4679 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda");
4684 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndi");
4689 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm");
4694 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn");
4699 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp");
4704 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx");
4709 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz");
4713 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ,
4717 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE,
4721 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE,
4725 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT,
4729 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT,
4733 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4735 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
4736 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
4741 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
4743 return Builder.CreateFPExt(Ops[0], Ty, "vcvt");
4749 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
4751 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt");
4767 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
4769 return Builder.CreateFPToUI(Ops[0], Ty);
4770 return Builder.CreateFPToSI(Ops[0], Ty);
4788 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta");
4806 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm");
4824 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn");
4842 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp");
4847 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx");
4855 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
4858 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
4859 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
4860 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]);
4868 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm");
4873 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm");
4878 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4879 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt");
4884 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit");
4896 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4897 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
4898 return Builder.CreateTrunc(Ops[0],
4910 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4911 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
4912 return Builder.CreateTrunc(Ops[0],
4924 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4925 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
4926 return Builder.CreateTrunc(Ops[0],
4938 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4939 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
4940 return Builder.CreateTrunc(Ops[0],
4949 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4950 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
4951 return Builder.CreateTrunc(Ops[0],
4960 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4961 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
4962 return Builder.CreateTrunc(Ops[0],
4971 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4972 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
4973 return Builder.CreateTrunc(Ops[0],
4982 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4983 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
4984 return Builder.CreateTrunc(Ops[0],
4993 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4994 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
4995 return Builder.CreateTrunc(Ops[0],
5004 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5005 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
5006 return Builder.CreateTrunc(Ops[0],
5015 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5016 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
5017 return Builder.CreateTrunc(Ops[0],
5026 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5027 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
5028 return Builder.CreateTrunc(Ops[0],
5037 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5038 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
5039 return Builder.CreateTrunc(Ops[0],
5048 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5049 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
5050 return Builder.CreateTrunc(Ops[0],
5059 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5060 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
5061 return Builder.CreateTrunc(Ops[0],
5070 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5071 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
5072 return Builder.CreateTrunc(Ops[0],
5081 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5082 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
5083 return Builder.CreateTrunc(Ops[0],
5092 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5093 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
5094 return Builder.CreateTrunc(Ops[0],
5103 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5104 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
5105 return Builder.CreateTrunc(Ops[0],
5114 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5115 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
5116 return Builder.CreateTrunc(Ops[0],
5120 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
5122 return Builder.CreateFMul(Ops[0], RHS);
5130 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5131 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
5132 return Builder.CreateTrunc(Ops[0],
5141 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5142 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
5150 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5151 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
5152 return Builder.CreateTrunc(Ops[0],
5161 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5162 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
5170 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5171 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
5172 return Builder.CreateTrunc(Ops[0],
5181 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5182 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
5190 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5191 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
5192 return Builder.CreateTrunc(Ops[0],
5201 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5202 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
5208 return EmitNeonCall(Intrin, Ops, "vsri_n");
5214 return EmitNeonCall(Intrin, Ops, "vsli_n");
5218 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5219 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
5220 return Builder.CreateAdd(Ops[0], Ops[1]);
5225 TmpOps.push_back(Ops[1]);
5226 TmpOps.push_back(Ops[2]);
5229 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
5230 return Builder.CreateAdd(Ops[0], tmp);
5241 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
5259 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN");
5260 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5261 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5262 return Builder.CreateStore(Ops[1], Ops[0]);
5287 SmallVector<Value *, 4> IntOps(Ops.begin()+1, Ops.end());
5288 IntOps.push_back(Ops[0]);
5293 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
5294 return Builder.CreateLoad(Ops[0]);
5297 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
5298 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
5299 return Builder.CreateStore(Ops[1], Ops[0]);
5302 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5304 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5305 Ops[0] = Builder.CreateLoad(Ops[0]);
5306 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane");
5311 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5312 Ops[0] = Builder.CreateLoad(Ops[0]);
5314 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI);
5315 return EmitNeonSplat(Ops[0], CI);
5319 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5320 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
5321 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5322 return Builder.CreateStore(Ops[1], Builder.CreateBitCast(Ops[0], Ty));
5326 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
5329 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
5330 Ops[0] = Builder.CreateBitCast(Ops[0],
5331 llvm::PointerType::getUnqual(Ops[1]->getType()));
5332 return Builder.CreateStore(Ops[1], Ops[0]);
5337 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
5340 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
5341 Ops[0] = Builder.CreateBitCast(Ops[0],
5342 llvm::PointerType::getUnqual(Ops[1]->getType()));
5343 return Builder.CreateStore(Ops[1], Ops[0]);
5348 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
5351 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
5352 Ops[0] = Builder.CreateBitCast(Ops[0],
5353 llvm::PointerType::getUnqual(Ops[1]->getType()));
5354 return Builder.CreateStore(Ops[1], Ops[0]);
5360 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
5363 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
5364 Ops[0] = Builder.CreateBitCast(Ops[0],
5365 llvm::PointerType::getUnqual(Ops[1]->getType()));
5366 return Builder.CreateStore(Ops[1], Ops[0]);
5372 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
5375 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
5376 Ops[0] = Builder.CreateBitCast(Ops[0],
5377 llvm::PointerType::getUnqual(Ops[1]->getType()));
5378 return Builder.CreateStore(Ops[1], Ops[0]);
5384 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
5387 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
5388 Ops[0] = Builder.CreateBitCast(Ops[0],
5389 llvm::PointerType::getUnqual(Ops[1]->getType()));
5390 return Builder.CreateStore(Ops[1], Ops[0]);
5394 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
5396 Ops.push_back(Ops[1]);
5397 Ops.erase(Ops.begin()+1);
5398 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5399 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5400 Ops[3] = Builder.CreateZExt(Ops[3],
5402 Ops[1] = Builder.CreateCall(F,
5403 ArrayRef<Value*>(Ops).slice(1), "vld2_lane");
5404 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5405 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5406 return Builder.CreateStore(Ops[1], Ops[0]);
5410 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
5412 Ops.push_back(Ops[1]);
5413 Ops.erase(Ops.begin()+1);
5414 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5415 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5416 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
5417 Ops[4] = Builder.CreateZExt(Ops[4],
5419 Ops[1] = Builder.CreateCall(F,
5420 ArrayRef<Value*>(Ops).slice(1), "vld3_lane");
5421 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5422 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5423 return Builder.CreateStore(Ops[1], Ops[0]);
5427 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
5429 Ops.push_back(Ops[1]);
5430 Ops.erase(Ops.begin()+1);
5431 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5432 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5433 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
5434 Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
5435 Ops[5] = Builder.CreateZExt(Ops[5],
5437 Ops[1] = Builder.CreateCall(F,
5438 ArrayRef<Value*>(Ops).slice(1), "vld4_lane");
5439 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5440 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5441 return Builder.CreateStore(Ops[1], Ops[0]);
5445 Ops.push_back(Ops[0]);
5446 Ops.erase(Ops.begin());
5447 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
5449 Ops, "");
5453 Ops.push_back(Ops[0]);
5454 Ops.erase(Ops.begin());
5455 Ops[2] = Builder.CreateZExt(Ops[2],
5457 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
5459 Ops, "");
5463 Ops.push_back(Ops[0]);
5464 Ops.erase(Ops.begin());
5465 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
5467 Ops, "");
5471 Ops.push_back(Ops[0]);
5472 Ops.erase(Ops.begin());
5473 Ops[3] = Builder.CreateZExt(Ops[3],
5475 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
5477 Ops, "");
5481 Ops.push_back(Ops[0]);
5482 Ops.erase(Ops.begin());
5483 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
5485 Ops, "");
5489 Ops.push_back(Ops[0]);
5490 Ops.erase(Ops.begin());
5491 Ops[4] = Builder.CreateZExt(Ops[4],
5493 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
5495 Ops, "");
5499 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
5500 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5501 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5510 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
5512 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn");
5519 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
5520 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5521 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5529 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
5531 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp");
5538 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
5539 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5540 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5549 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
5551 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip");
5558 Ops, "vtbl1");
5562 Ops, "vtbl2");
5566 Ops, "vtbl3");
5570 Ops, "vtbl4");
5574 Ops, "vtbx1");
5578 Ops, "vtbx2");
5582 Ops, "vtbx3");
5586 Ops, "vtbx4");
5591 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd");
5596 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd");
5602 BuildVector(ArrayRef<llvm::Value*> Ops) {
5603 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
5606 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
5607 AllConstants &= isa<Constant>(Ops[i]);
5612 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
5613 CstOps.push_back(cast<Constant>(Ops[i]));
5619 llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size()));
5621 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
5622 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i));
5629 SmallVector<Value*, 4> Ops;
5640 Ops.push_back(EmitScalarExpr(E->getArg(i)));
5649 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
5665 return Builder.CreateBitCast(BuildVector(Ops),
5668 return Builder.CreateExtractElement(Ops[0],
5669 llvm::ConstantInt::get(Ops[1]->getType(), 0));
5672 Builder.CreateStore(Ops[0], Tmp);
5688 Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast");
5693 Ops[1] = Builder.CreateExtractElement(Ops[1], Idx, "extract");
5696 Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy);
5697 return Builder.CreateStore(Ops[1], Ops[0]);
5700 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
5710 return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
5719 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
5720 Ops[1] = llvm::ConstantInt::get(VecTy, (shiftVal-8) * 8);
5724 return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr");
5731 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
5741 return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
5749 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
5750 Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8);
5754 return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr");
5761 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
5779 return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
5787 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
5788 Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8);
5792 return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr");
5810 Value *BC = Builder.CreateBitCast(Ops[0],
5811 llvm::PointerType::getUnqual(Ops[1]->getType()),
5813 StoreInst *SI = Builder.CreateStore(Ops[1], BC);
5841 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast");
5843 return Builder.CreateCall(F, Ops, name);
5875 Builder.CreateStore(Builder.CreateExtractValue(Call, 0), Ops[0]);
5881 Builder.CreateStore(Ops[0], VecTmp);
5891 SmallVector<Value*, 4> Ops;
5894 Ops.push_back(EmitScalarExpr(E->getArg(i)));
5910 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
5912 Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]);
5913 Ops.pop_back();
5940 return Builder.CreateCall(F, Ops, "");
5950 Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy);
5951 Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]);
5952 Ops.pop_back();
5973 return Builder.CreateCall(F, Ops, "");
