Lines Matching full:cgf
69 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
73 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
75 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
140 void Enter(CodeGenFunction &CGF) override {
143 auto PartIdLVal = CGF.EmitLoadOfPointerLValue(
144 CGF.GetAddrOfLocalVar(PartIDVar),
146 auto *Res = CGF.EmitLoadOfScalar(PartIdLVal, SourceLocation());
147 auto *DoneBB = CGF.createBasicBlock(".untied.done.");
148 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
149 CGF.EmitBlock(DoneBB);
150 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
151 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
152 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
153 CGF.Builder.GetInsertBlock());
154 emitUntiedSwitch(CGF);
157 void emitUntiedSwitch(CodeGenFunction &CGF) const {
159 auto PartIdLVal = CGF.EmitLoadOfPointerLValue(
160 CGF.GetAddrOfLocalVar(PartIDVar),
162 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
164 UntiedCodeGen(CGF);
166 CGF.getJumpDestInCurrentScope(".untied.next.");
167 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
168 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
169 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
170 CGF.Builder.GetInsertBlock());
171 CGF.EmitBranchThroughCleanup(CurPoint);
172 CGF.EmitBlock(CurPoint.getBlock());
192 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
197 void emitUntiedSwitch(CodeGenFunction &CGF) override {
198 Action.emitUntiedSwitch(CGF);
271 void emitUntiedSwitch(CodeGenFunction &CGF) override {
273 OuterRegionInfo->emitUntiedSwitch(CGF);
327 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
328 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
331 PrivScope(CGF) {
347 PrivScope.addPrivate(VD, [&CGF, &DRE]() -> Address {
348 return CGF.EmitLValue(&DRE).getAddress();
362 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
386 CodeGenFunction &CGF;
395 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
397 : CGF(CGF) {
399 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
400 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
401 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
402 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
403 CGF.LambdaThisCaptureField = nullptr;
409 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
410 delete CGF.CapturedStmtInfo;
411 CGF.CapturedStmtInfo = OldCSI;
412 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
413 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
677 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
678 if (!CGF.HaveInsertPoint())
680 Action->Exit(CGF);
686 void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
687 CodeGenFunction::RunCleanupsScope Scope(CGF);
689 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
690 Callback(CodeGen, CGF, *PrePostAction);
693 Callback(CodeGen, CGF, Action);
697 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
698 return CGF.EmitLoadOfPointerLValue(
699 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
703 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
704 if (!CGF.HaveInsertPoint())
711 CGF.EHStack.pushTerminate();
712 CodeGen(CGF);
713 CGF.EHStack.popTerminate();
717 CodeGenFunction &CGF) {
718 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
760 CodeGenFunction CGF(CGM);
763 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args);
764 CodeGenFunction::OMPPrivateScope Scope(CGF);
765 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
766 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() -> Address {
767 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
770 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
771 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() -> Address {
772 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
776 CGF.EmitIgnoredExpr(CombinerInitializer);
778 CGF.FinishFunction();
783 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
807 if (CGF) {
808 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
818 emitUserDefinedReduction(/*CGF=*/nullptr, D);
834 static Address createIdentFieldGEP(CodeGenFunction &CGF, Address Addr,
838 return CGF.Builder.CreateStructGEP(Addr, Field, Offset, Name);
847 CodeGenFunction CGF(CGM, true);
857 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
858 return CGF.GenerateOpenMPCapturedStmtFunction(*CS);
866 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
868 auto *ThreadID = getThreadID(CGF, D.getLocStart());
869 auto *UpLoc = emitUpdateLocation(CGF, D.getLocStart());
872 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
875 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
884 CodeGenFunction CGF(CGM, true);
888 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
889 auto *Res = CGF.GenerateCapturedStmtFunction(*CS);
926 llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
935 assert(CGF.CurFn && "No function in current CodeGenFunction.");
938 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
940 LocValue = Address(I->second.DebugLoc, getIdentAlign(CGF.CGM));
946 Address AI = CGF.CreateTempAlloca(IdentTy, getIdentAlign(CGF.CGM),
948 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
952 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
953 CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
954 CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
955 CGM.getSize(getIdentSize(CGF.CGM)));
959 Address PSource = createIdentFieldGEP(CGF, LocValue, IdentField_PSource);
966 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
969 dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) {
973 OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
977 CGF.Builder.CreateStore(OMPDebugLoc, PSource);
984 llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
986 assert(CGF.CurFn && "No function in current CodeGenFunction.");
991 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
998 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1001 auto LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1002 ThreadID = CGF.EmitLoadOfLValue(LVal, Loc).getScalarVal();
1005 if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
1006 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1017 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1018 CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
1020 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
1021 emitUpdateLocation(CGF, Loc));
1022 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1027 void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1028 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1029 if (OpenMPLocThreadIDMap.count(CGF.CurFn))
1030 OpenMPLocThreadIDMap.erase(CGF.CurFn);
1031 if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1032 for(auto *D : FunctionUDRMap[CGF.CurFn]) {
1035 FunctionUDRMap.erase(CGF.CurFn);
1733 Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
1742 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1743 CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
1747 return Address(CGF.EmitRuntimeCall(
1753 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
1757 auto OMPLoc = emitUpdateLocation(CGF, Loc);
1758 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
1763 CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
1766 CGF.EmitRuntimeCall(
1772 bool PerformInit, CodeGenFunction *CGF) {
1865 if (!CGF) {
1880 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1892 static void emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond,
1895 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
1900 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
1902 ThenGen(CGF);
1904 ElseGen(CGF);
1910 auto ThenBlock = CGF.createBasicBlock("omp_if.then");
1911 auto ElseBlock = CGF.createBasicBlock("omp_if.else");
1912 auto ContBlock = CGF.createBasicBlock("omp_if.end");
1913 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
1916 CGF.EmitBlock(ThenBlock);
1917 ThenGen(CGF);
1918 CGF.EmitBranch(ContBlock);
1921 (void)ApplyDebugLocation::CreateEmpty(CGF);
1922 CGF.EmitBlock(ElseBlock);
1923 ElseGen(CGF);
1925 (void)ApplyDebugLocation::CreateEmpty(CGF);
1926 CGF.EmitBranch(ContBlock);
1928 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
1931 void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
1935 if (!CGF.HaveInsertPoint())
1937 auto *RTLoc = emitUpdateLocation(CGF, Loc);
1938 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF,
1941 auto &RT = CGF.CGM.getOpenMPRuntime();
1944 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
1945 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
1951 CGF.EmitRuntimeCall(RTLFn, RealArgs);
1953 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF,
1955 auto &RT = CGF.CGM.getOpenMPRuntime();
1956 auto ThreadID = RT.getThreadID(CGF, Loc);
1960 CGF.EmitRuntimeCall(
1964 auto ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
1966 CGF.CreateTempAlloca(CGF.Int32Ty, CharUnits::fromQuantity(4),
1968 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
1973 CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs);
1976 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
1977 CGF.EmitRuntimeCall(
1982 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
1985 ThenRCG(CGF);
1995 Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
1998 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2000 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
2002 auto ThreadID = getThreadID(CGF, Loc);
2004 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
2005 auto ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
2006 CGF.EmitStoreOfScalar(ThreadID,
2007 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
2053 void Enter(CodeGenFunction &CGF) override {
2054 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
2056 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
2057 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
2058 ContBlock = CGF.createBasicBlock("omp_if.end");
2060 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2061 CGF.EmitBlock(ThenBlock);
2064 void Done(CodeGenFunction &CGF) {
2066 CGF.EmitBranch(ContBlock);
2067 CGF.EmitBlock(ContBlock, true);
2069 void Exit(CodeGenFunction &CGF) override {
2070 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
2075 void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
2083 if (!CGF.HaveInsertPoint())
2085 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2090 EnterArgs.push_back(CGF.Builder.CreateIntCast(
2091 CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false));
2098 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
2101 void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
2104 if (!CGF.HaveInsertPoint())
2111 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2116 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
2117 Action.Done(CGF);
2120 void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
2122 if (!CGF.HaveInsertPoint())
2126 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2128 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args);
2129 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2130 Region->emitUntiedSwitch(CGF);
2133 void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
2136 if (!CGF.HaveInsertPoint())
2142 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2147 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
2152 static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
2156 CGF.Builder.CreateConstArrayGEP(Array, Index, CGF.getPointerSize());
2157 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
2159 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
2160 Addr = CGF.Builder.CreateElementBitCast(
2161 Addr, CGF.ConvertTypeForMem(Var->getType()));
2183 CodeGenFunction CGF(CGM);
2184 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args);
2187 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2188 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
2189 ArgsType), CGF.getPointerAlign());
2190 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2191 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
2192 ArgsType), CGF.getPointerAlign());
2199 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
2202 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
2206 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
2208 CGF.FinishFunction();
2212 void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
2219 if (!CGF.HaveInsertPoint())
2238 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
2239 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
2242 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2247 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
2250 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
2252 Action.Done(CGF);
2262 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
2264 Address Elem = CGF.Builder.CreateConstArrayGEP(
2265 CopyprivateList, I, CGF.getPointerSize());
2266 CGF.Builder.CreateStore(
2267 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2268 CGF.EmitLValue(CopyprivateVars[I]).getPointer(), CGF.VoidPtrTy),
2274 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
2276 auto *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
2278 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
2279 CGF.VoidPtrTy);
2280 auto *DidItVal = CGF.Builder.CreateLoad(DidIt);
2282 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
2283 getThreadID(CGF, Loc), // i32 <gtid>
2289 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args);
2293 void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
2296 if (!CGF.HaveInsertPoint())
2303 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2308 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
2311 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
2314 void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
2317 if (!CGF.HaveInsertPoint())
2334 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
2335 getThreadID(CGF, Loc)};
2337 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
2339 auto *Result = CGF.EmitRuntimeCall(
2345 auto *ExitBB = CGF.createBasicBlock(".cancel.exit");
2346 auto *ContBB = CGF.createBasicBlock(".cancel.continue");
2347 auto *Cmp = CGF.Builder.CreateIsNotNull(Result);
2348 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2349 CGF.EmitBlock(ExitBB);
2352 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
2353 CGF.EmitBranchThroughCleanup(CancelDestination);
2354 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
2359 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args);
2448 void CGOpenMPRuntime::emitForDispatchInit(CodeGenFunction &CGF,
2454 if (!CGF.HaveInsertPoint())
2469 Chunk = CGF.Builder.getIntN(IVSize, 1);
2471 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2472 CGF.Builder.getInt32(addMonoNonMonoModifier(
2474 CGF.Builder.getIntN(IVSize, 0), // Lower
2476 CGF.Builder.getIntN(IVSize, 1), // Stride
2479 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
2483 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2488 if (!CGF.HaveInsertPoint())
2508 Chunk = CGF.Builder.getIntN(IVSize, 1);
2517 UpdateLocation, ThreadId, CGF.Builder.getInt32(addMonoNonMonoModifier(
2523 CGF.Builder.getIntN(IVSize, 1), // Incr
2526 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
2529 void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
2538 auto *UpdatedLocation = emitUpdateLocation(CGF, Loc);
2539 auto *ThreadId = getThreadID(CGF, Loc);
2541 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
2547 CodeGenFunction &CGF, SourceLocation Loc,
2552 auto *UpdatedLocation = emitUpdateLocation(CGF, Loc);
2553 auto *ThreadId = getThreadID(CGF, Loc);
2555 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
2561 void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
2563 if (!CGF.HaveInsertPoint())
2566 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2567 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
2571 void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
2575 if (!CGF.HaveInsertPoint())
2578 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2579 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
2582 llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
2592 emitUpdateLocation(CGF, Loc),
2593 getThreadID(CGF, Loc),
2600 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
2601 return CGF.EmitScalarConversion(
2602 Call, CGF.getContext().getIntTypeForBitwidth(32, /* Signed */ true),
2603 CGF.getContext().BoolTy, Loc);
2606 void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
2609 if (!CGF.HaveInsertPoint())
2613 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2614 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
2615 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads),
2619 void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
2622 if (!CGF.HaveInsertPoint())
2649 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2651 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args);
2654 void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
2656 if (!CGF.HaveInsertPoint())
2659 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
2660 emitUpdateLocation(CGF, Loc));
2769 CodeGenFunction CGF(CGM);
2775 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FI, Args, SourceLocation());
2776 Codegen(CGF);
2777 CGF.FinishFunction();
2874 [&](CodeGenFunction &CGF, PrePostActionTy &) {
2875 CGF.EmitCallOrInvoke(createRuntimeFunction(OMPRTL__tgt_unregister_lib),
2880 [&](CodeGenFunction &CGF, PrePostActionTy &) {
2881 CGF.EmitCallOrInvoke(createRuntimeFunction(OMPRTL__tgt_register_lib),
2883 CGM.getCXXABI().registerGlobalDtor(CGF, RegUnregVar, UnRegFn, Desc);
3285 CodeGenFunction CGF(CGM);
3286 CGF.disableDebugInfo();
3287 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args);
3294 auto *GtidParam = CGF.EmitLoadOfScalar(
3295 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
3296 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3297 CGF.GetAddrOfLocalVar(&TaskTypeArg),
3302 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
3305 auto PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
3309 auto SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
3310 auto *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3311 CGF.EmitLoadOfLValue(SharedsLVal, Loc).getScalarVal(),
3312 CGF.ConvertTypeForMem(SharedsPtrTy));
3317 auto PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
3318 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3319 PrivatesLVal.getPointer(), CGF.VoidPtrTy);
3321 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
3325 CGF.Builder
3327 TDBase.getAddress(), CGF.VoidPtrTy)
3333 auto LBLVal = CGF.EmitLValueForField(Base, *LBFI);
3334 auto *LBParam = CGF.EmitLoadOfLValue(LBLVal, Loc).getScalarVal();
3336 auto UBLVal = CGF.EmitLValueForField(Base, *UBFI);
3337 auto *UBParam = CGF.EmitLoadOfLValue(UBLVal, Loc).getScalarVal();
3339 auto StLVal = CGF.EmitLValueForField(Base, *StFI);
3340 auto *StParam = CGF.EmitLoadOfLValue(StLVal, Loc).getScalarVal();
3342 auto LILVal = CGF.EmitLValueForField(Base, *LIFI);
3343 auto *LIParam = CGF.EmitLoadOfLValue(LILVal, Loc).getScalarVal();
3351 CGF.EmitCallOrInvoke(TaskFunction, CallArgs);
3352 CGF.EmitStoreThroughLValue(
3353 RValue::get(CGF.Builder.getInt32(/*C=*/0)),
3354 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
3355 CGF.FinishFunction();
3381 CodeGenFunction CGF(CGM);
3382 CGF.disableDebugInfo();
3383 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
3386 LValue Base = CGF.EmitLoadOfPointerLValue(
3387 CGF.GetAddrOfLocalVar(&TaskTypeArg),
3392 Base = CGF.EmitLValueForField(Base, *FI);
3396 auto FieldLValue = CGF.EmitLValueForField(Base, Field);
3397 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
3400 CGF.FinishFunction();
3469 CodeGenFunction CGF(CGM);
3470 CGF.disableDebugInfo();
3471 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
3475 LValue Base = CGF.EmitLoadOfPointerLValue(
3476 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
3481 auto FieldLVal = CGF.EmitLValueForField(Base, Field);
3483 auto RefLVal = CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
3484 auto RefLoadLVal = CGF.EmitLoadOfPointerLValue(
3486 CGF.EmitStoreOfScalar(FieldLVal.getPointer(), RefLoadLVal);
3489 CGF.FinishFunction();
3499 static void emitPrivatesInit(CodeGenFunction &CGF,
3506 auto &C = CGF.getContext();
3508 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
3511 SrcBase = CGF.MakeAddrLValue(
3512 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3513 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
3523 !CGF.isTrivialInitializer(Init)))) {
3524 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
3528 auto SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
3529 SharedRefLValue = CGF.MakeAddrLValue(
3535 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
3537 CGF.EmitAggregateAssign(PrivateLValue.getAddress(),
3542 CGF.EmitOMPAggregateAssign(
3544 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
3547 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3553 CGF, &CapturesInfo);
3554 CGF.EmitAnyExprToMem(Init, DestElement,
3560 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3565 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3566 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
3570 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
3577 static bool checkInitIsRequired(CodeGenFunction &CGF,
3584 !CGF.isTrivialInitializer(Init));
3626 CodeGenFunction CGF(CGM);
3627 CGF.disableDebugInfo();
3628 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args);
3630 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3631 CGF.GetAddrOfLocalVar(&DstArg),
3636 LValue Base = CGF.EmitLValueForField(
3638 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
3639 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
3640 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
3641 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
3648 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3649 CGF.GetAddrOfLocalVar(&SrcArg),
3651 LValue Base = CGF.EmitLValueForField(
3654 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
3658 CGF.getNaturalTypeAlignment(SharedsTy));
3660 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3662 CGF.FinishFunction();
3682 CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
3736 auto *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
3738 auto *KmpTaskTWithPrivatesTySize = CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
3752 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3788 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
3789 CGF.Builder.getInt32(FinalFlag),
3790 CGF.Builder.getInt32(/*C=*/0))
3791 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
3792 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
3794 llvm::Value *AllocArgs[] = {emitUpdateLocation(CGF, Loc),
3795 getThreadID(CGF, Loc), TaskFlags,
3797 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3799 auto *NewTask = CGF.EmitRuntimeCall(
3801 auto *NewTaskNewTaskTTy = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3803 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
3806 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
3812 Address(CGF.EmitLoadOfScalar(
3813 CGF.EmitLValueForField(
3817 CGF.getNaturalTypeAlignment(SharedsTy));
3818 CGF.EmitAggregateCopy(KmpTaskSharedsPtr, Shareds, SharedsTy);
3823 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
3827 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
3843 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
3844 LValue DestructorsLV = CGF.EmitLValueForField(
3846 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3852 LValue Data2LV = CGF.EmitLValueForField(
3854 LValue PriorityLV = CGF.EmitLValueForField(
3856 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
3866 void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
3872 if (!CGF.HaveInsertPoint())
3876 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
3893 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
3911 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
3914 auto Addr = CGF.EmitLValue(E);
3919 CGF.EmitOMPArraySectionExpr(ASE, /*LowerBound=*/false);
3921 CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(), /*Idx0=*/1);
3923 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGM.SizeTy);
3924 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy);
3925 Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
3927 Size = CGF.getTypeSize(Ty);
3928 auto Base = CGF.MakeAddrLValue(
3929 CGF.Builder.CreateConstArrayGEP(DependenciesArray, i, DependencySize),
3932 auto BaseAddrLVal = CGF.EmitLValueForField(
3934 CGF.EmitStoreOfScalar(
3935 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGF.IntPtrTy),
3938 auto LenLVal = CGF.EmitLValueForField(
3940 CGF.EmitStoreOfScalar(Size, LenLVal);
3957 auto FlagsLVal = CGF.EmitLValueForField(
3959 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
3962 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3963 CGF.Builder.CreateStructGEP(DependenciesArray, 0, CharUnits::Zero()),
3964 CGF.VoidPtrTy);
3973 auto *ThreadID = getThreadID(CGF, Loc);
3974 auto *UpLoc = emitUpdateLocation(CGF, Loc);
3981 DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies);
3983 DepTaskArgs[5] = CGF.Builder.getInt32(0);
3984 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
3988 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
3991 auto PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
3992 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
3995 CGF.EmitRuntimeCall(
3998 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task),
4003 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
4004 Region->emitUntiedSwitch(CGF);
4011 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
4013 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
4014 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4017 NumDependencies, &DepWaitTaskArgs](CodeGenFunction &CGF,
4019 auto &RT = CGF.CGM.getOpenMPRuntime();
4020 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
4026 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps),
4030 CodeGenFunction &CGF, PrePostActionTy &Action) {
4031 Action.Enter(CGF);
4033 CGF.EmitCallOrInvoke(TaskEntry, OutlinedFnArgs);
4045 RCG(CGF);
4049 emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
4052 ThenRCG(CGF);
4056 void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
4062 if (!CGF.HaveInsertPoint())
4065 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4071 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4072 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4075 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
4078 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
4080 LValue LBLVal = CGF.EmitLValueForField(
4085 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(),
4087 LValue UBLVal = CGF.EmitLValueForField(
4092 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(),
4094 LValue StLVal = CGF.EmitLValueForField(
4099 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(),
4104 UBLVal.getPointer(), CGF.EmitLoadOfScalar(StLVal, SourceLocation()),
4105 llvm::ConstantInt::getSigned(CGF.IntTy, Data.Nogroup ? 1 : 0),
4107 CGF.IntTy, Data.Schedule.getPointer()
4111 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
4113 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
4115 ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Result.TaskDupFn,
4116 CGF.VoidPtrTy)
4117 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
4118 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs);
4131 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
4133 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
4139 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
4140 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
4144 auto NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
4149 auto LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
4151 auto BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
4152 auto DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
4154 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
4155 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4158 auto EntryBB = CGF.Builder.GetInsertBlock();
4159 CGF.EmitBlock(BodyBB);
4161 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
4163 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
4170 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
4178 CodeGenFunction::OMPPrivateScope Scope(CGF);
4182 RedOpGen(CGF, XExpr, EExpr, UpExpr);
4186 auto LHSElementNext = CGF.Builder.CreateConstGEP1_32(
4188 auto RHSElementNext = CGF.Builder.CreateConstGEP1_32(
4192 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
4193 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
4194 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
4195 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
4198 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
4204 static void emitReductionCombiner(CodeGenFunction &CGF,
4212 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
4214 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
4215 CGF.EmitIgnoredExpr(ReductionOp);
4218 CGF.EmitIgnoredExpr(ReductionOp);
4242 CodeGenFunction CGF(CGM);
4243 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args);
4247 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4248 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
4249 ArgsType), CGF.getPointerAlign());
4250 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4251 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
4252 ArgsType), CGF.getPointerAlign());
4257 CodeGenFunction::OMPPrivateScope Scope(CGF);
4263 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
4267 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
4274 CGF.Builder.CreateConstArrayGEP(LHS, Idx, CGF.getPointerSize());
4275 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
4276 auto *VLA = CGF.getContext().getAsVariableArrayType(PrivTy);
4279 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
4280 CGF.EmitVariablyModifiedType(PrivTy);
4293 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
4294 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
4295 emitReductionCombiner(CGF, E);
4299 emitReductionCombiner(CGF, E);
4305 CGF.FinishFunction();
4309 static void emitSingleReductionCombiner(CodeGenFunction &CGF,
4319 CGF, PrivateRef->getType(), LHSVar, RHSVar,
4320 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
4321 emitReductionCombiner(CGF, ReductionOp);
4325 emitReductionCombiner(CGF, ReductionOp);
4328 void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
4334 if (!CGF.HaveInsertPoint())
4374 CodeGenFunction::RunCleanupsScope Scope(CGF);
4379 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
4401 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
4406 CGF.Builder.CreateConstArrayGEP(ReductionList, Idx, CGF.getPointerSize());
4407 CGF.Builder.CreateStore(
4408 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4409 CGF.EmitLValue(RHSExprs[I]).getPointer(), CGF.VoidPtrTy),
4414 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx,
4415 CGF.getPointerSize());
4416 llvm::Value *Size = CGF.Builder.CreateIntCast(
4417 CGF.getVLASize(
4418 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
4420 CGF.SizeTy, /*isSigned=*/false);
4421 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
4428 CGM, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates,
4436 auto *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
4437 auto *ThreadId = getThreadID(CGF, Loc);
4438 auto *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
4440 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList.getPointer(),
4441 CGF.VoidPtrTy);
4445 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
4451 auto Res = CGF.EmitRuntimeCall(
4457 auto *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");
4458 auto *SwInst = CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);
4466 auto *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");
4467 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
4468 CGF.EmitBlock(Case1BB);
4477 CodeGenFunction &CGF, PrePostActionTy &Action) {
4482 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
4496 RCG(CGF);
4498 CGF.EmitBranch(DefaultBB);
4505 auto *Case2BB = CGF.createBasicBlock(".omp.reduction.case2");
4506 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
4507 CGF.EmitBlock(Case2BB);
4510 CodeGenFunction &CGF, PrePostActionTy &Action) {
4544 Loc](CodeGenFunction &CGF, const Expr *XExpr,
4546 LValue X = CGF.EmitLValue(XExpr);
4549 E = CGF.EmitAnyExpr(EExpr);
4550 CGF.EmitOMPAtomicSimpleUpdateExpr(
4553 [&CGF, UpExpr, VD, IPriv, Loc](RValue XRValue) {
4554 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4556 VD, [&CGF, VD, XRValue, Loc]() -> Address {
4557 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
4558 CGF.emitOMPSimpleStore(
4559 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
4564 return CGF.EmitAnyExpr(UpExpr);
4570 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
4574 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
4577 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
4579 auto &RT = CGF.CGM.getOpenMPRuntime();
4581 CGF, ".atomic_reduction",
4582 [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
4583 Action.Enter(CGF);
4584 emitReductionCombiner(CGF, E);
4591 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
4594 CritRedGen(CGF, nullptr, nullptr, nullptr);
4613 AtomicRCG(CGF);
4615 AtomicRCG(CGF);
4617 CGF.EmitBranch(DefaultBB);
4618 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);
4621 void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
4623 if (!CGF.HaveInsertPoint())
4627 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
4629 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskwait), Args);
4630 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
4631 Region->emitUntiedSwitch(CGF);
4634 void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,
4638 if (!CGF.HaveInsertPoint())
4640 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel);
4641 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
4670 CodeGenFunction &CGF, SourceLocation Loc,
4672 if (!CGF.HaveInsertPoint())
4677 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
4680 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
4681 CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
4683 auto *Result = CGF.EmitRuntimeCall(
4689 auto *ExitBB = CGF.createBasicBlock(".cancel.exit");
4690 auto *ContBB = CGF.createBasicBlock(".cancel.continue");
4691 auto *Cmp = CGF.Builder.CreateIsNotNull(Result);
4692 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
4693 CGF.EmitBlock(ExitBB);
4695 emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false);
4698 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
4699 CGF.EmitBranchThroughCleanup(CancelDest);
4700 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
4705 void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
4708 if (!CGF.HaveInsertPoint())
4713 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
4714 auto &&ThenGen = [Loc, CancelRegion, OMPRegionInfo](CodeGenFunction &CGF,
4716 auto &RT = CGF.CGM.getOpenMPRuntime();
4718 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
4719 CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
4721 auto *Result = CGF.EmitRuntimeCall(
4727 auto *ExitBB = CGF.createBasicBlock(".cancel.exit");
4728 auto *ContBB = CGF.createBasicBlock(".cancel.continue");
4729 auto *Cmp = CGF.Builder.CreateIsNotNull(Result);
4730 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
4731 CGF.EmitBlock(ExitBB);
4733 RT.emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false);
4736 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
4737 CGF.EmitBranchThroughCleanup(CancelDest);
4738 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
4741 emitOMPIfClause(CGF, IfCond, ThenGen,
4745 ThenRCG(CGF);
4814 CodeGenFunction CGF(CGM, true);
4816 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
4818 OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS);
4864 CodeGenFunction &CGF,
4867 assert(!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the "
4891 CGOpenMPInnerExprInfo CGInfo(CGF, CS);
4892 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
4893 llvm::Value *NumTeams = CGF.EmitScalarExpr(NTE->getNumTeams());
4894 return CGF.Builder.CreateIntCast(NumTeams, CGF.Int32Ty,
4900 return CGF.Builder.getInt32(0);
4913 CodeGenFunction &CGF,
4916 assert(!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the "
4940 CGOpenMPInnerExprInfo CGInfo(CGF, CS);
4941 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
4942 llvm::Value *ThreadLimit = CGF.EmitScalarExpr(TLE->getThreadLimit());
4943 return CGF.Builder.CreateIntCast(ThreadLimit, CGF.Int32Ty,
4949 return CGF.Builder.getInt32(0);
4999 CodeGenFunction &CGF;
5013 // on CGF.getTypeSize(E->getType()).
5022 return CGF.getTypeSize(BaseTy);
5026 ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType());
5030 ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType());
5038 auto *LengthVal = CGF.EmitScalarExpr(OAE->getLength());
5040 CGF.Builder.CreateIntCast(LengthVal, CGF.SizeTy, /*isSigned=*/false);
5041 return CGF.Builder.CreateNUWMul(LengthVal, ElemSize);
5043 return CGF.getTypeSize(ExprTy);
5119 if (!Length->EvaluateAsInt(ConstLength, CGF.getContext()))
5265 BP = CGF.EmitScalarExpr(ME->getBase());
5269 BP = CGF.EmitLValue(cast<DeclRefExpr>(I->getAssociatedExpression()))
5277 auto PtrAddr = CGF.MakeNaturalAlignAddrLValue(
5279 BP = CGF.EmitLoadOfPointerLValue(PtrAddr.getAddress(),
5328 auto *LB = CGF.EmitLValue(I->getAssociatedExpression()).getPointer();
5375 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
5376 : Directive(Dir), CGF(CGF) {
5493 CurSizes.push_back(CGF.getTypeSize(PtrTy->getPointeeType()));
5504 CurSizes.push_back(CGF.getTypeSize(RI.getType()));
5509 CurSizes.push_back(llvm::Constant::getNullValue(CGF.SizeTy));
5519 CurSizes.push_back(CGF.getTypeSize(ElementType));
5551 emitOffloadingArrays(CodeGenFunction &CGF, llvm::Value *&BasePointersArray,
5558 auto &CGM = CGF.CGM;
5559 auto &Ctx = CGF.getContext();
5579 CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer();
5581 CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer();
5591 CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer();
5612 llvm::ConstantDataArray::get(CGF.Builder.getContext(), MapTypes);
5623 BPVal = CGF.Builder.CreateBitCast(BPVal, CGM.VoidPtrTy);
5627 BPVal = CGF.Builder.CreateIntToPtr(BPVal, CGM.VoidPtrTy);
5629 llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32(
5633 CGF.Builder.CreateStore(BPVal, BPAddr);
5637 PVal = CGF.Builder.CreateBitCast(PVal, CGM.VoidPtrTy);
5641 PVal = CGF.Builder.CreateIntToPtr(PVal, CGM.VoidPtrTy);
5643 llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32(
5647 CGF.Builder.CreateStore(PVal, PAddr);
5650 llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32(
5655 CGF.Builder.CreateStore(
5656 CGF.Builder.CreateIntCast(Sizes[i], CGM.SizeTy, /*isSigned=*/true),
5665 CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg,
5670 auto &CGM = CGF.CGM;
5672 BasePointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
5675 PointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
5679 SizesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
5682 MapTypesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
5695 void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF,
5701 if (!CGF.HaveInsertPoint())
5706 auto &Ctx = CGF.getContext();
5721 MappableExprsHandler MEHandler(D, CGF);
5742 CurSizes.push_back(CGF.getTypeSize(RI->getType()));
5775 auto OffloadError = CGF.MakeAddrLValue(
5776 CGF.CreateMemTemp(OffloadErrorQType, ".run_host_version"),
5778 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.Int32Ty),
5784 &D](CodeGenFunction &CGF, PrePostActionTy &) {
5785 auto &RT = CGF.CGM.getOpenMPRuntime();
5791 emitOffloadingArrays(CGF, BasePointersArray, PointersArray, SizesArray,
5794 emitOffloadingArraysArgument(CGF, BasePointersArray, PointersArray,
5814 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
5815 CGF.Int32Ty, /*isSigned=*/true);
5817 DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF);
5820 llvm::Value *PointerNum = CGF.Builder.getInt32(BasePointers.size());
5825 auto *NumTeams = emitNumTeamsClauseForTargetDirective(RT, CGF, D);
5826 auto *ThreadLimit = emitThreadLimitClauseForTargetDirective(RT, CGF, D);
5841 Return = CGF.EmitRuntimeCall(
5847 Return = CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__tgt_target),
5851 CGF.EmitStoreOfScalar(Return, OffloadError);
5855 auto &&ElseGen = [OffloadError](CodeGenFunction &CGF, PrePostActionTy &) {
5856 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.Int32Ty, /*V=*/-1u),
5866 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
5869 ThenRCG(CGF);
5873 ElseRCG(CGF);
5877 auto OffloadFailedBlock = CGF.createBasicBlock("omp_offload.failed");
5878 auto OffloadContBlock = CGF.createBasicBlock("omp_offload.cont");
5879 auto OffloadErrorVal = CGF.EmitLoadOfScalar(OffloadError, SourceLocation());
5880 auto Failed = CGF.Builder.CreateIsNotNull(OffloadErrorVal);
5881 CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock);
5883 CGF.EmitBlock(OffloadFailedBlock);
5884 CGF.Builder.CreateCall(OutlinedFn, KernelArgs);
5885 CGF.EmitBranch(OffloadContBlock);
5887 CGF.EmitBlock(OffloadContBlock, /*IsFinished=*/true);
6006 void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF,
6011 if (!CGF.HaveInsertPoint())
6014 auto *RTLoc = emitUpdateLocation(CGF, Loc);
6015 CodeGenFunction::RunCleanupsScope Scope(CGF);
6020 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
6021 CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())};
6027 CGF.EmitRuntimeCall(RTLFn, RealArgs);
6030 void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
6034 if (!CGF.HaveInsertPoint())
6037 auto *RTLoc = emitUpdateLocation(CGF, Loc);
6041 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams),
6042 CGF.CGM.Int32Ty, /* isSigned = */ true)
6043 : CGF.Builder.getInt32(0);
6047 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit),
6048 CGF.CGM.Int32Ty, /* isSigned = */ true)
6049 : CGF.Builder.getInt32(0);
6052 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,
6054 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_teams),
6058 void CGOpenMPRuntime::emitTargetDataCalls(CodeGenFunction &CGF,
6064 if (!CGF.HaveInsertPoint())
6076 auto &&BeginThenGen = [&D, &CGF, &BasePointersArray, &PointersArray,
6078 &NumOfPtrs](CodeGenFunction &CGF, PrePostActionTy &) {
6086 MappableExprsHandler MCHandler(D, CGF);
6091 emitOffloadingArrays(CGF, BasePointersArray, PointersArray, SizesArray,
6099 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg,
6107 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
6108 CGF.Int32Ty, /*isSigned=*/true);
6110 DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF);
6113 auto *PointerNum = CGF.Builder.getInt32(NumOfPtrs);
6118 auto &RT = CGF.CGM.getOpenMPRuntime();
6119 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__tgt_target_data_begin),
6124 auto &&EndThenGen = [&CGF, &BasePointersArray, &PointersArray, &SizesArray,
6126 &NumOfPtrs](CodeGenFunction &CGF, PrePostActionTy &) {
6134 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg,
6142 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
6143 CGF.Int32Ty, /*isSigned=*/true);
6145 DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF);
6148 auto *PointerNum = CGF.Builder.getInt32(NumOfPtrs);
6153 auto &RT = CGF.CGM.getOpenMPRuntime();
6154 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__tgt_target_data_end),
6160 auto &&ElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {};
6163 emitOMPIfClause(CGF, IfCond, BeginThenGen, ElseGen);
6166 BeginThenRCG(CGF);
6169 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data, CodeGen);
6172 emitOMPIfClause(CGF, IfCond, EndThenGen, ElseGen);
6175 EndThenRCG(CGF);
6180 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
6182 if (!CGF.HaveInsertPoint())
6191 auto &&ThenGen = [&D, &CGF, Device](CodeGenFunction &CGF, PrePostActionTy &) {
6199 MappableExprsHandler MEHandler(D, CGF);
6208 emitOffloadingArrays(CGF, BasePointersArrayArg, PointersArrayArg,
6212 CGF, BasePointersArrayArg, PointersArrayArg, SizesArrayArg,
6219 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
6220 CGF.Int32Ty, /*isSigned=*/true);
6222 DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF);
6225 auto *PointerNum = CGF.Builder.getInt32(BasePointers.size());
6231 auto &RT = CGF.CGM.getOpenMPRuntime();
6249 CGF.EmitRuntimeCall(RT.createRuntimeFunction(RTLFn), OffloadingArgs);
6254 auto &&ElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {};
6257 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
6260 ThenGenRCG(CGF);
6505 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
6506 if (!CGF.HaveInsertPoint())
6508 CGF.EmitRuntimeCall(RTLFn, Args);
6513 void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
6515 if (!CGF.HaveInsertPoint())
6537 Address DimsAddr = CGF.CreateMemTemp(KmpDimTy, "dims");
6538 CGF.EmitNullInitialization(DimsAddr, KmpDimTy);
6541 LValue DimsLVal = CGF.MakeAddrLValue(DimsAddr, KmpDimTy);
6544 CGF.EmitLValueForField(DimsLVal, *std::next(RD->field_begin(), UpperFD));
6545 llvm::Value *NumIterVal = CGF.EmitScalarConversion(
6546 CGF.EmitScalarExpr(D.getNumIterations()), D.getNumIterations()->getType(),
6548 CGF.EmitStoreOfScalar(NumIterVal, UpperLVal);
6551 CGF.EmitLValueForField(DimsLVal, *std::next(RD->field_begin(), StrideFD));
6552 CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1),
6557 llvm::Value *Args[] = {emitUpdateLocation(CGF, D.getLocStart()),
6558 getThreadID(CGF, D.getLocStart()),
6560 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6564 CGF.EmitRuntimeCall(RTLFn, Args);
6566 emitUpdateLocation(CGF, D.getLocEnd()), getThreadID(CGF, D.getLocEnd())};
6568 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn,
6572 void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
6578 llvm::Value *CntVal = CGF.EmitScalarConversion(CGF.EmitScalarExpr(CounterVal),
6581 Address CntAddr = CGF.CreateMemTemp(Int64Ty, ".cnt.addr");
6582 CGF.EmitStoreOfScalar(CntVal, CntAddr, /*Volatile=*/false, Int64Ty);
6583 llvm::Value *Args[] = {emitUpdateLocation(CGF, C->getLocStart()),
6584 getThreadID(CGF, C->getLocStart()),
6593 CGF.EmitRuntimeCall(RTLFn, Args);
