Lines Matching full:cgf
30 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
36 CGF.EmitVarDecl(cast<VarDecl>(*I));
39 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
40 CGF.EmitAutoVarCleanups(Emission);
49 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
50 return CGF.LambdaCaptureFields.lookup(VD) ||
51 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
52 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
56 OMPLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S,
58 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
59 InlinedShareds(CGF) {
60 emitPreInitStmt(CGF, S);
68 isCapturedVar(CGF, VD) ||
69 (CGF.CapturedStmtInfo &&
73 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
74 return CGF.EmitLValue(&DRE).getAddress();
87 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
91 CGF.EmitVarDecl(cast<VarDecl>(*I));
97 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
98 : CodeGenFunction::RunCleanupsScope(CGF) {
99 emitPreInitStmt(CGF, S);
172 static Address castValueFromUintptr(CodeGenFunction &CGF, QualType DstType,
175 ASTContext &Ctx = CGF.getContext();
177 auto *CastedPtr = CGF.EmitScalarConversion(
181 CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
189 TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name) + ".ref");
190 auto TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
191 CGF.EmitScalarInit(RefVal, TmpLVal);
397 static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
404 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
411 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
418 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
419 CGF.EmitIgnoredExpr(InitOp);
421 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
423 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
425 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
427 switch (CGF.getEvaluationKind(Ty)) {
429 InitRVal = CGF.EmitLoadOfLValue(LV, SourceLocation());
433 RValue::getComplex(CGF.EmitLoadOfComplex(LV, SourceLocation()));
440 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
441 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
451 static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
460 auto NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
462 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
465 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
472 auto DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
474 auto BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
475 auto DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
477 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
478 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
481 auto EntryBB = CGF.Builder.GetInsertBlock();
482 CGF.EmitBlock(BodyBB);
484 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
489 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
496 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
505 CodeGenFunction::RunCleanupsScope InitScope(CGF);
507 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
510 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
516 auto SrcElementNext = CGF.Builder.CreateConstGEP1_32(
518 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
522 auto DestElementNext = CGF.Builder.CreateConstGEP1_32(
526 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
527 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
528 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
531 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
886 static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
893 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
894 Tmp = CGF.CreateMemTemp(BaseTy);
896 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
905 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
907 CGF.Builder.CreateStore(Addr, Tmp);
913 static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
917 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
919 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy);
921 BaseLV = CGF.EmitLoadOfReferenceLValue(BaseLV.getAddress(),
926 return CGF.MakeAddrLValue(
928 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
929 BaseLV.getPointer(), CGF.ConvertTypeForMem(ElTy)->getPointerTo()),
1166 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1168 if (!CGF.HaveInsertPoint())
1174 if (auto *Cond = CondGen(CGF)) {
1177 auto *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1178 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1179 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1180 CGF.EmitBlock(ThenBB);
1183 CGF.EmitIgnoredExpr(PostUpdate);
1187 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1190 static void emitCommonOMPParallelDirective(CodeGenFunction &CGF,
1195 auto OutlinedFn = CGF.CGM.getOpenMPRuntime().
1199 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1200 auto NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1202 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1203 CGF, NumThreads, NumThreadsClause->getLocStart());
1206 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1207 CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1208 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart());
1219 OMPLexicalScope Scope(CGF, S);
1221 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1222 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn,
1228 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1229 OMPPrivateScope PrivateScope(CGF);
1230 bool Copyins = CGF.EmitOMPCopyinClause(S);
1231 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1236 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1237 CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
1240 CGF.EmitOMPPrivateClause(S, PrivateScope);
1241 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1243 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
1244 CGF.EmitOMPReductionClauseFinal(S);
1391 static void emitAlignedClause(CodeGenFunction &CGF,
1393 if (!CGF.HaveInsertPoint())
1399 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1409 CGF.getContext()
1410 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
1417 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1418 CGF.EmitAlignmentAssumption(PtrValue, Alignment);
1456 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1459 if (!CGF.HaveInsertPoint())
1462 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1463 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1467 CGF.EmitIgnoredExpr(I);
1471 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1508 static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
1511 if (!CGF.HaveInsertPoint())
1514 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1517 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1522 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
1524 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1527 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1531 CGF.LoopStack.setParallel(false);
1590 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1591 OMPLoopScope PreInitScope(CGF, S);
1603 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1607 auto *ThenBlock = CGF.createBasicBlock("simd.if.then");
1608 ContBlock = CGF.createBasicBlock("simd.if.end");
1609 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1610 CGF.getProfileCount(&S));
1611 CGF.EmitBlock(ThenBlock);
1612 CGF.incrementProfileCounter(&S);
1618 CGF.EmitVarDecl(*IVDecl);
1619 CGF.EmitIgnoredExpr(S.getInit());
1625 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1627 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
1630 CGF.EmitOMPSimdInit(S);
1632 emitAlignedClause(CGF, S);
1633 CGF.EmitOMPLinearClauseInit(S);
1635 OMPPrivateScope LoopScope(CGF);
1636 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1637 CGF.EmitOMPLinearClause(S, LoopScope);
1638 CGF.EmitOMPPrivateClause(S, LoopScope);
1639 CGF.EmitOMPReductionClauseInit(S, LoopScope);
1641 CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1643 CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1645 [&S](CodeGenFunction &CGF) {
1646 CGF.EmitOMPLoopBody(S, JumpDest());
1647 CGF.EmitStopPoint(&S);
1650 CGF.EmitOMPSimdFinal(
1654 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1655 CGF.EmitOMPReductionClauseFinal(S);
1657 CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1659 CGF.EmitOMPLinearClauseFinal(
1663 CGF.EmitBranch(ContBlock);
1664 CGF.EmitBlock(ContBlock, true);
1732 [&S, LoopExit](CodeGenFunction &CGF) {
1733 CGF.EmitOMPLoopBody(S, LoopExit);
1734 CGF.EmitStopPoint(&S);
1736 [Ordered, IVSize, IVSigned, Loc](CodeGenFunction &CGF) {
1738 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(
1739 CGF, Loc, IVSize, IVSigned);
1873 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1874 OMPLoopScope PreInitScope(CGF, S);
1875 CGF.EmitStmt(
1885 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1886 OMPLoopScope PreInitScope(CGF, S);
1887 CGF.EmitStmt(
1897 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1898 OMPLoopScope PreInitScope(CGF, S);
1899 CGF.EmitStmt(
1909 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1910 OMPLoopScope PreInitScope(CGF, S);
1911 CGF.EmitStmt(
1917 static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
1920 CGF.EmitVarDecl(*VDecl);
1921 return CGF.EmitLValue(Helper);
2056 [&S, LoopExit](CodeGenFunction &CGF) {
2057 CGF.EmitOMPLoopBody(S, LoopExit);
2058 CGF.EmitStopPoint(&S);
2078 [&](CodeGenFunction &CGF) -> llvm::Value * {
2079 return CGF.Builder.CreateIsNotNull(
2080 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2086 *this, S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2087 return CGF.Builder.CreateIsNotNull(
2088 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2096 EmitOMPLinearClauseFinal(S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2097 return CGF.Builder.CreateIsNotNull(
2098 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2111 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2113 HasLastprivates = CGF.EmitOMPWorksharingLoop(S);
2129 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2131 HasLastprivates = CGF.EmitOMPWorksharingLoop(S);
2144 static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
2147 auto LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2149 CGF.EmitScalarInit(Init, LVal);
2157 auto &&CodeGen = [&S, Stmt, CS, &HasLastprivates](CodeGenFunction &CGF,
2159 auto &C = CGF.CGM.getContext();
2162 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2163 CGF.Builder.getInt32(0));
2164 auto *GlobalUBVal = CS != nullptr ? CGF.Builder.getInt32(CS->size() - 1)
2165 : CGF.Builder.getInt32(0);
2167 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2168 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2169 CGF.Builder.getInt32(1));
2170 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2171 CGF.Builder.getInt32(0));
2173 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2175 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2177 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2185 auto BodyGen = [Stmt, CS, &S, &IV](CodeGenFunction &CGF) {
2197 auto *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2198 auto *SwitchStmt = CGF.Builder.CreateSwitch(
2199 CGF.EmitLoadOfLValue(IV, S.getLocStart()).getScalarVal(), ExitBB,
2204 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2205 CGF.EmitBlock(CaseBB);
2206 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2207 CGF.EmitStmt(SubStmt);
2208 CGF.EmitBranch(ExitBB);
2212 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2213 CGF.EmitBlock(CaseBB);
2214 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2215 CGF.EmitStmt(Stmt);
2216 CGF.EmitBranch(ExitBB);
2218 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2221 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2222 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2226 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2227 CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
2230 CGF.EmitOMPPrivateClause(S, LoopScope);
2231 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2232 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2238 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2239 CGF, S.getLocStart(), ScheduleKind, /*IVSize=*/32,
2243 auto *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart());
2244 auto *MinUBGlobalUB = CGF.Builder.CreateSelect(
2245 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2246 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2248 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getLocStart()), IV);
2250 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2253 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocStart());
2254 CGF.EmitOMPReductionClauseFinal(S);
2257 CGF, S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2258 return CGF.Builder.CreateIsNotNull(
2259 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2264 CGF.EmitOMPLastprivateClauseFinal(
2266 CGF.Builder.CreateIsNotNull(
2267 CGF.EmitLoadOfScalar(IL, S.getLocStart())));
2301 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2302 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2327 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2328 Action.Enter(CGF);
2329 OMPPrivateScope SingleScope(CGF);
2330 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2331 CGF.EmitOMPPrivateClause(S, SingleScope);
2333 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2351 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2352 Action.Enter(CGF);
2353 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2360 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2361 Action.Enter(CGF);
2362 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2377 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2378 CGF.EmitOMPWorksharingLoop(S);
2387 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2388 CGF.EmitOMPWorksharingLoop(S);
2397 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2398 CGF.EmitSections(S);
2491 CodeGenFunction &CGF, PrePostActionTy &Action) {
2493 OMPPrivateScope Scope(CGF);
2496 auto *CopyFn = CGF.Builder.CreateLoad(
2497 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)));
2498 auto *PrivatesPtr = CGF.Builder.CreateLoad(
2499 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)));
2506 Address PrivatePtr = CGF.CreateMemTemp(
2507 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2514 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2522 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2527 CGF.EmitRuntimeCall(CopyFn, CallArgs);
2532 /*RefersToEnclosingVariableOrCapture=*/CGF.CapturedStmtInfo->lookup(
2535 Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2536 return CGF.EmitLValue(&DRE).getAddress();
2540 Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2541 CGF.getContext().getDeclAlign(Pair.first));
2547 Action.Enter(CGF);
2548 BodyGen(CGF);
2574 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
2575 CGF.EmitStmt(CS->getCapturedStmt());
2578 IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
2580 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getLocStart(), S, OutlinedFn,
2602 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2603 Action.Enter(CGF);
2604 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2711 [&S, LoopExit](CodeGenFunction &CGF) {
2712 CGF.EmitOMPLoopBody(S, LoopExit);
2713 CGF.EmitStopPoint(&S);
2738 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2739 CGF.EmitOMPDistributeLoop(S);
2748 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
2750 CGF.CapturedStmtInfo = &CapStmtInfo;
2751 auto *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
2763 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
2768 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
2770 CGF.EmitNounwindRuntimeCall(OutlinedFn, CapturedVars);
2772 Action.Enter(CGF);
2773 CGF.EmitStmt(
2781 static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
2784 assert(CGF.hasScalarEvaluationKind(DestType) &&
2788 ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType,
2790 : CGF.EmitComplexToScalarConversion(Val.getComplexVal(), SrcType,
2795 convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
2797 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
2803 auto ScalarVal = CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
2811 ComplexVal.first = CGF.EmitScalarConversion(
2813 ComplexVal.second = CGF.EmitScalarConversion(
2819 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
2822 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
2824 CGF.EmitAtomicStore(RVal, LVal,
2849 static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
2855 LValue XLValue = CGF.EmitLValue(X);
2856 LValue VLValue = CGF.EmitLValue(V);
2858 ? CGF.EmitLoadOfLValue(XLValue, Loc)
2859 : CGF.EmitAtomicLoad(
2869 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
2870 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
2873 static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
2878 emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
2884 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
2887 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
2892 auto &Context = CGF.CGM.getContext();
2971 UpdateVal = CGF.Builder.CreateIntCast(
2975 auto *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3003 static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3017 LValue XLValue = CGF.EmitLValue(X);
3018 RValue ExprRValue = CGF.EmitAnyExpr(E);
3026 [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) -> RValue {
3027 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3028 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3029 return CGF.EmitAnyExpr(UE);
3031 (void)CGF.EmitOMPAtomicSimpleUpdateExpr(
3038 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3041 static RValue convertToType(CodeGenFunction &CGF, RValue Value,
3044 switch (CGF.getEvaluationKind(ResType)) {
3047 convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3049 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3058 static void EmitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3066 LValue VLValue = CGF.EmitLValue(V);
3067 LValue XLValue = CGF.EmitLValue(X);
3068 RValue ExprRValue = CGF.EmitAnyExpr(E);
3088 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3090 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3091 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3092 RValue Res = CGF.EmitAnyExpr(UE);
3096 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3106 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3107 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3108 NewVVal = CGF.EmitAnyExpr(UE);
3114 ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
3116 auto &&Gen = [&CGF, &NewVVal, ExprRValue](RValue XRValue) -> RValue {
3121 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3130 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
3136 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3139 static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
3146 EmitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3149 EmitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3153 EmitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3156 EmitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3232 auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
3234 CGF.EmitStopPoint(CS);
3235 EmitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
3249 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3250 OMPPrivateScope PrivateScope(CGF);
3251 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
3252 CGF.EmitOMPPrivateClause(S, PrivateScope);
3255 Action.Enter(CGF);
3256 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3317 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
3322 auto OutlinedFn = CGF.CGM.getOpenMPRuntime().
3333 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
3337 OMPLexicalScope Scope(CGF, S);
3339 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3340 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getLocStart(), OutlinedFn,
3346 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3347 OMPPrivateScope PrivateScope(CGF);
3348 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
3349 CGF.EmitOMPPrivateClause(S, PrivateScope);
3351 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3389 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3390 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3467 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
3472 VDecl, [&CGF, PVD]() -> Address { return CGF.GetAddrOfLocalVar(PVD); });
3506 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
3518 OMPLoopScope PreInitScope(CGF, S);
3519 if (CGF
3523 auto *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
3524 ContBlock = CGF.createBasicBlock("taskloop.if.end");
3525 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
3526 CGF.getProfileCount(&S));
3527 CGF.EmitBlock(ThenBlock);
3528 CGF.incrementProfileCounter(&S);
3532 CGF.EmitOMPSimdInit(S);
3534 OMPPrivateScope LoopScope(CGF);
3542 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
3544 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
3546 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
3547 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
3549 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
3550 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
3555 CGF.EmitVarDecl(*IVDecl);
3556 CGF.EmitIgnoredExpr(S.getInit());
3562 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3564 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
3567 CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
3569 [&S](CodeGenFunction &CGF) {
3570 CGF.EmitOMPLoopBody(S, JumpDest());
3571 CGF.EmitStopPoint(&S);
3576 CGF.EmitBranch(ContBlock);
3577 CGF.EmitBlock(ContBlock, true);
3581 CGF.EmitOMPLastprivateClauseFinal(
3583 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
3584 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
3589 IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
3591 auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &) {
3592 OMPLoopScope PreInitScope(CGF, S);
3593 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getLocStart(), S,
3597 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
