1 //===- Parsing, selection, and construction of pass pipelines -------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 /// \file 10 /// 11 /// This file provides the implementation of the PassBuilder based on our 12 /// static pass registry as well as related functionality. It also provides 13 /// helpers to aid in analyzing, debugging, and testing passes and pass 14 /// pipelines. 15 /// 16 //===----------------------------------------------------------------------===// 17 18 #include "llvm/Passes/PassBuilder.h" 19 #include "llvm/ADT/StringSwitch.h" 20 #include "llvm/Analysis/AliasAnalysis.h" 21 #include "llvm/Analysis/AliasAnalysisEvaluator.h" 22 #include "llvm/Analysis/AssumptionCache.h" 23 #include "llvm/Analysis/BasicAliasAnalysis.h" 24 #include "llvm/Analysis/BlockFrequencyInfo.h" 25 #include "llvm/Analysis/BranchProbabilityInfo.h" 26 #include "llvm/Analysis/CFGPrinter.h" 27 #include "llvm/Analysis/CFLAndersAliasAnalysis.h" 28 #include "llvm/Analysis/CFLSteensAliasAnalysis.h" 29 #include "llvm/Analysis/CGSCCPassManager.h" 30 #include "llvm/Analysis/CallGraph.h" 31 #include "llvm/Analysis/DemandedBits.h" 32 #include "llvm/Analysis/DependenceAnalysis.h" 33 #include "llvm/Analysis/DominanceFrontier.h" 34 #include "llvm/Analysis/GlobalsModRef.h" 35 #include "llvm/Analysis/IVUsers.h" 36 #include "llvm/Analysis/LazyCallGraph.h" 37 #include "llvm/Analysis/LazyValueInfo.h" 38 #include "llvm/Analysis/LoopAccessAnalysis.h" 39 #include "llvm/Analysis/LoopInfo.h" 40 #include "llvm/Analysis/MemoryDependenceAnalysis.h" 41 #include "llvm/Analysis/MemorySSA.h" 42 #include "llvm/Analysis/ModuleSummaryAnalysis.h" 43 #include "llvm/Analysis/OptimizationRemarkEmitter.h" 44 #include "llvm/Analysis/PhiValues.h" 45 #include "llvm/Analysis/PostDominators.h" 46 #include "llvm/Analysis/ProfileSummaryInfo.h" 47 #include "llvm/Analysis/RegionInfo.h" 48 #include "llvm/Analysis/ScalarEvolution.h" 49 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" 50 #include "llvm/Analysis/ScopedNoAliasAA.h" 51 #include "llvm/Analysis/TargetLibraryInfo.h" 52 #include "llvm/Analysis/TargetTransformInfo.h" 53 #include "llvm/Analysis/TypeBasedAliasAnalysis.h" 54 #include "llvm/CodeGen/PreISelIntrinsicLowering.h" 55 #include "llvm/CodeGen/UnreachableBlockElim.h" 56 #include "llvm/IR/Dominators.h" 57 #include "llvm/IR/IRPrintingPasses.h" 58 #include "llvm/IR/PassManager.h" 59 #include "llvm/IR/Verifier.h" 60 #include "llvm/Support/Debug.h" 61 #include "llvm/Support/Regex.h" 62 #include "llvm/Target/TargetMachine.h" 63 #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h" 64 #include "llvm/Transforms/Instrumentation/CGProfile.h" 65 #include "llvm/Transforms/IPO/AlwaysInliner.h" 66 #include "llvm/Transforms/IPO/ArgumentPromotion.h" 67 #include "llvm/Transforms/IPO/CalledValuePropagation.h" 68 #include "llvm/Transforms/IPO/ConstantMerge.h" 69 #include "llvm/Transforms/IPO/CrossDSOCFI.h" 70 #include "llvm/Transforms/IPO/DeadArgumentElimination.h" 71 #include "llvm/Transforms/IPO/ElimAvailExtern.h" 72 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h" 73 #include "llvm/Transforms/IPO/FunctionAttrs.h" 74 #include "llvm/Transforms/IPO/FunctionImport.h" 75 #include "llvm/Transforms/IPO/GlobalDCE.h" 76 #include "llvm/Transforms/IPO/GlobalOpt.h" 77 #include "llvm/Transforms/IPO/GlobalSplit.h" 78 #include "llvm/Transforms/IPO/InferFunctionAttrs.h" 79 #include "llvm/Transforms/IPO/Inliner.h" 80 #include "llvm/Transforms/IPO/Internalize.h" 81 #include "llvm/Transforms/IPO/LowerTypeTests.h" 82 #include "llvm/Transforms/IPO/PartialInlining.h" 83 #include "llvm/Transforms/IPO/SCCP.h" 84 #include "llvm/Transforms/IPO/SampleProfile.h" 85 #include "llvm/Transforms/IPO/StripDeadPrototypes.h" 86 #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h" 87 #include "llvm/Transforms/IPO/WholeProgramDevirt.h" 88 #include "llvm/Transforms/InstCombine/InstCombine.h" 89 #include "llvm/Transforms/Instrumentation/BoundsChecking.h" 90 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h" 91 #include "llvm/Transforms/Instrumentation/InstrProfiling.h" 92 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" 93 #include "llvm/Transforms/Scalar/ADCE.h" 94 #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h" 95 #include "llvm/Transforms/Scalar/BDCE.h" 96 #include "llvm/Transforms/Scalar/CallSiteSplitting.h" 97 #include "llvm/Transforms/Scalar/ConstantHoisting.h" 98 #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h" 99 #include "llvm/Transforms/Scalar/DCE.h" 100 #include "llvm/Transforms/Scalar/DeadStoreElimination.h" 101 #include "llvm/Transforms/Scalar/DivRemPairs.h" 102 #include "llvm/Transforms/Scalar/EarlyCSE.h" 103 #include "llvm/Transforms/Scalar/Float2Int.h" 104 #include "llvm/Transforms/Scalar/GVN.h" 105 #include "llvm/Transforms/Scalar/GuardWidening.h" 106 #include "llvm/Transforms/Scalar/IVUsersPrinter.h" 107 #include "llvm/Transforms/Scalar/IndVarSimplify.h" 108 #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h" 109 #include "llvm/Transforms/Scalar/InstSimplifyPass.h" 110 #include "llvm/Transforms/Scalar/JumpThreading.h" 111 #include "llvm/Transforms/Scalar/LICM.h" 112 #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h" 113 #include "llvm/Transforms/Scalar/LoopDataPrefetch.h" 114 #include "llvm/Transforms/Scalar/LoopDeletion.h" 115 #include "llvm/Transforms/Scalar/LoopDistribute.h" 116 #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h" 117 #include "llvm/Transforms/Scalar/LoopInstSimplify.h" 118 #include "llvm/Transforms/Scalar/LoopLoadElimination.h" 119 #include "llvm/Transforms/Scalar/LoopPassManager.h" 120 #include "llvm/Transforms/Scalar/LoopPredication.h" 121 #include "llvm/Transforms/Scalar/LoopRotation.h" 122 #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h" 123 #include "llvm/Transforms/Scalar/LoopSink.h" 124 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h" 125 #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h" 126 #include "llvm/Transforms/Scalar/LoopUnrollPass.h" 127 #include "llvm/Transforms/Scalar/LowerAtomic.h" 128 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h" 129 #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h" 130 #include "llvm/Transforms/Scalar/MemCpyOptimizer.h" 131 #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h" 132 #include "llvm/Transforms/Scalar/NaryReassociate.h" 133 #include "llvm/Transforms/Scalar/NewGVN.h" 134 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h" 135 #include "llvm/Transforms/Scalar/Reassociate.h" 136 #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h" 137 #include "llvm/Transforms/Scalar/SCCP.h" 138 #include "llvm/Transforms/Scalar/SROA.h" 139 #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h" 140 #include "llvm/Transforms/Scalar/SimplifyCFG.h" 141 #include "llvm/Transforms/Scalar/Sink.h" 142 #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h" 143 #include "llvm/Transforms/Scalar/SpeculativeExecution.h" 144 #include "llvm/Transforms/Scalar/TailRecursionElimination.h" 145 #include "llvm/Transforms/Utils/AddDiscriminators.h" 146 #include "llvm/Transforms/Utils/BreakCriticalEdges.h" 147 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h" 148 #include "llvm/Transforms/Utils/LCSSA.h" 149 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h" 150 #include "llvm/Transforms/Utils/LoopSimplify.h" 151 #include "llvm/Transforms/Utils/LowerInvoke.h" 152 #include "llvm/Transforms/Utils/Mem2Reg.h" 153 #include "llvm/Transforms/Utils/NameAnonGlobals.h" 154 #include "llvm/Transforms/Utils/SymbolRewriter.h" 155 #include "llvm/Transforms/Vectorize/LoopVectorize.h" 156 #include "llvm/Transforms/Vectorize/SLPVectorizer.h" 157 158 using namespace llvm; 159 160 static cl::opt<unsigned> MaxDevirtIterations("pm-max-devirt-iterations", 161 cl::ReallyHidden, cl::init(4)); 162 static cl::opt<bool> 163 RunPartialInlining("enable-npm-partial-inlining", cl::init(false), 164 cl::Hidden, cl::ZeroOrMore, 165 cl::desc("Run Partial inlinining pass")); 166 167 static cl::opt<bool> 168 RunNewGVN("enable-npm-newgvn", cl::init(false), 169 cl::Hidden, cl::ZeroOrMore, 170 cl::desc("Run NewGVN instead of GVN")); 171 172 static cl::opt<bool> EnableEarlyCSEMemSSA( 173 "enable-npm-earlycse-memssa", cl::init(true), cl::Hidden, 174 cl::desc("Enable the EarlyCSE w/ MemorySSA pass for the new PM (default = on)")); 175 176 static cl::opt<bool> EnableGVNHoist( 177 "enable-npm-gvn-hoist", cl::init(false), cl::Hidden, 178 cl::desc("Enable the GVN hoisting pass for the new PM (default = off)")); 179 180 static cl::opt<bool> EnableGVNSink( 181 "enable-npm-gvn-sink", cl::init(false), cl::Hidden, 182 cl::desc("Enable the GVN hoisting pass for the new PM (default = off)")); 183 184 static cl::opt<bool> EnableUnrollAndJam( 185 "enable-npm-unroll-and-jam", cl::init(false), cl::Hidden, 186 cl::desc("Enable the Unroll and Jam pass for the new PM (default = off)")); 187 188 static cl::opt<bool> EnableSyntheticCounts( 189 "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore, 190 cl::desc("Run synthetic function entry count generation " 191 "pass")); 192 193 static Regex DefaultAliasRegex( 194 "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$"); 195 196 static bool isOptimizingForSize(PassBuilder::OptimizationLevel Level) { 197 switch (Level) { 198 case PassBuilder::O0: 199 case PassBuilder::O1: 200 case PassBuilder::O2: 201 case PassBuilder::O3: 202 return false; 203 204 case PassBuilder::Os: 205 case PassBuilder::Oz: 206 return true; 207 } 208 llvm_unreachable("Invalid optimization level!"); 209 } 210 211 namespace { 212 213 /// No-op module pass which does nothing. 214 struct NoOpModulePass { 215 PreservedAnalyses run(Module &M, ModuleAnalysisManager &) { 216 return PreservedAnalyses::all(); 217 } 218 static StringRef name() { return "NoOpModulePass"; } 219 }; 220 221 /// No-op module analysis. 222 class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> { 223 friend AnalysisInfoMixin<NoOpModuleAnalysis>; 224 static AnalysisKey Key; 225 226 public: 227 struct Result {}; 228 Result run(Module &, ModuleAnalysisManager &) { return Result(); } 229 static StringRef name() { return "NoOpModuleAnalysis"; } 230 }; 231 232 /// No-op CGSCC pass which does nothing. 233 struct NoOpCGSCCPass { 234 PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &, 235 LazyCallGraph &, CGSCCUpdateResult &UR) { 236 return PreservedAnalyses::all(); 237 } 238 static StringRef name() { return "NoOpCGSCCPass"; } 239 }; 240 241 /// No-op CGSCC analysis. 242 class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> { 243 friend AnalysisInfoMixin<NoOpCGSCCAnalysis>; 244 static AnalysisKey Key; 245 246 public: 247 struct Result {}; 248 Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) { 249 return Result(); 250 } 251 static StringRef name() { return "NoOpCGSCCAnalysis"; } 252 }; 253 254 /// No-op function pass which does nothing. 255 struct NoOpFunctionPass { 256 PreservedAnalyses run(Function &F, FunctionAnalysisManager &) { 257 return PreservedAnalyses::all(); 258 } 259 static StringRef name() { return "NoOpFunctionPass"; } 260 }; 261 262 /// No-op function analysis. 263 class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> { 264 friend AnalysisInfoMixin<NoOpFunctionAnalysis>; 265 static AnalysisKey Key; 266 267 public: 268 struct Result {}; 269 Result run(Function &, FunctionAnalysisManager &) { return Result(); } 270 static StringRef name() { return "NoOpFunctionAnalysis"; } 271 }; 272 273 /// No-op loop pass which does nothing. 274 struct NoOpLoopPass { 275 PreservedAnalyses run(Loop &L, LoopAnalysisManager &, 276 LoopStandardAnalysisResults &, LPMUpdater &) { 277 return PreservedAnalyses::all(); 278 } 279 static StringRef name() { return "NoOpLoopPass"; } 280 }; 281 282 /// No-op loop analysis. 283 class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> { 284 friend AnalysisInfoMixin<NoOpLoopAnalysis>; 285 static AnalysisKey Key; 286 287 public: 288 struct Result {}; 289 Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) { 290 return Result(); 291 } 292 static StringRef name() { return "NoOpLoopAnalysis"; } 293 }; 294 295 AnalysisKey NoOpModuleAnalysis::Key; 296 AnalysisKey NoOpCGSCCAnalysis::Key; 297 AnalysisKey NoOpFunctionAnalysis::Key; 298 AnalysisKey NoOpLoopAnalysis::Key; 299 300 } // End anonymous namespace. 301 302 void PassBuilder::invokePeepholeEPCallbacks( 303 FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { 304 for (auto &C : PeepholeEPCallbacks) 305 C(FPM, Level); 306 } 307 308 void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) { 309 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \ 310 MAM.registerPass([&] { return CREATE_PASS; }); 311 #include "PassRegistry.def" 312 313 for (auto &C : ModuleAnalysisRegistrationCallbacks) 314 C(MAM); 315 } 316 317 void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) { 318 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \ 319 CGAM.registerPass([&] { return CREATE_PASS; }); 320 #include "PassRegistry.def" 321 322 for (auto &C : CGSCCAnalysisRegistrationCallbacks) 323 C(CGAM); 324 } 325 326 void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) { 327 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ 328 FAM.registerPass([&] { return CREATE_PASS; }); 329 #include "PassRegistry.def" 330 331 for (auto &C : FunctionAnalysisRegistrationCallbacks) 332 C(FAM); 333 } 334 335 void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) { 336 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \ 337 LAM.registerPass([&] { return CREATE_PASS; }); 338 #include "PassRegistry.def" 339 340 for (auto &C : LoopAnalysisRegistrationCallbacks) 341 C(LAM); 342 } 343 344 FunctionPassManager 345 PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, 346 ThinLTOPhase Phase, 347 bool DebugLogging) { 348 assert(Level != O0 && "Must request optimizations!"); 349 FunctionPassManager FPM(DebugLogging); 350 351 // Form SSA out of local memory accesses after breaking apart aggregates into 352 // scalars. 353 FPM.addPass(SROA()); 354 355 // Catch trivial redundancies 356 FPM.addPass(EarlyCSEPass(EnableEarlyCSEMemSSA)); 357 358 // Hoisting of scalars and load expressions. 359 if (EnableGVNHoist) 360 FPM.addPass(GVNHoistPass()); 361 362 // Global value numbering based sinking. 363 if (EnableGVNSink) { 364 FPM.addPass(GVNSinkPass()); 365 FPM.addPass(SimplifyCFGPass()); 366 } 367 368 // Speculative execution if the target has divergent branches; otherwise nop. 369 FPM.addPass(SpeculativeExecutionPass()); 370 371 // Optimize based on known information about branches, and cleanup afterward. 372 FPM.addPass(JumpThreadingPass()); 373 FPM.addPass(CorrelatedValuePropagationPass()); 374 FPM.addPass(SimplifyCFGPass()); 375 if (Level == O3) 376 FPM.addPass(AggressiveInstCombinePass()); 377 FPM.addPass(InstCombinePass()); 378 379 if (!isOptimizingForSize(Level)) 380 FPM.addPass(LibCallsShrinkWrapPass()); 381 382 invokePeepholeEPCallbacks(FPM, Level); 383 384 // For PGO use pipeline, try to optimize memory intrinsics such as memcpy 385 // using the size value profile. Don't perform this when optimizing for size. 386 if (PGOOpt && !PGOOpt->ProfileUseFile.empty() && 387 !isOptimizingForSize(Level)) 388 FPM.addPass(PGOMemOPSizeOpt()); 389 390 FPM.addPass(TailCallElimPass()); 391 FPM.addPass(SimplifyCFGPass()); 392 393 // Form canonically associated expression trees, and simplify the trees using 394 // basic mathematical properties. For example, this will form (nearly) 395 // minimal multiplication trees. 396 FPM.addPass(ReassociatePass()); 397 398 // Add the primary loop simplification pipeline. 399 // FIXME: Currently this is split into two loop pass pipelines because we run 400 // some function passes in between them. These can and should be removed 401 // and/or replaced by scheduling the loop pass equivalents in the correct 402 // positions. But those equivalent passes aren't powerful enough yet. 403 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still 404 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to 405 // fully replace `SimplifyCFGPass`, and the closest to the other we have is 406 // `LoopInstSimplify`. 407 LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging); 408 409 // Simplify the loop body. We do this initially to clean up after other loop 410 // passes run, either when iterating on a loop or on inner loops with 411 // implications on the outer loop. 412 LPM1.addPass(LoopInstSimplifyPass()); 413 LPM1.addPass(LoopSimplifyCFGPass()); 414 415 // Rotate Loop - disable header duplication at -Oz 416 LPM1.addPass(LoopRotatePass(Level != Oz)); 417 LPM1.addPass(LICMPass()); 418 LPM1.addPass(SimpleLoopUnswitchPass()); 419 LPM2.addPass(IndVarSimplifyPass()); 420 LPM2.addPass(LoopIdiomRecognizePass()); 421 422 for (auto &C : LateLoopOptimizationsEPCallbacks) 423 C(LPM2, Level); 424 425 LPM2.addPass(LoopDeletionPass()); 426 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO 427 // because it changes IR to makes profile annotation in back compile 428 // inaccurate. 429 if (Phase != ThinLTOPhase::PreLink || 430 !PGOOpt || PGOOpt->SampleProfileFile.empty()) 431 LPM2.addPass(LoopFullUnrollPass(Level)); 432 433 for (auto &C : LoopOptimizerEndEPCallbacks) 434 C(LPM2, Level); 435 436 // We provide the opt remark emitter pass for LICM to use. We only need to do 437 // this once as it is immutable. 438 FPM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>()); 439 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1), DebugLogging)); 440 FPM.addPass(SimplifyCFGPass()); 441 FPM.addPass(InstCombinePass()); 442 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2), DebugLogging)); 443 444 // Eliminate redundancies. 445 if (Level != O1) { 446 // These passes add substantial compile time so skip them at O1. 447 FPM.addPass(MergedLoadStoreMotionPass()); 448 if (RunNewGVN) 449 FPM.addPass(NewGVNPass()); 450 else 451 FPM.addPass(GVN()); 452 } 453 454 // Specially optimize memory movement as it doesn't look like dataflow in SSA. 455 FPM.addPass(MemCpyOptPass()); 456 457 // Sparse conditional constant propagation. 458 // FIXME: It isn't clear why we do this *after* loop passes rather than 459 // before... 460 FPM.addPass(SCCPPass()); 461 462 // Delete dead bit computations (instcombine runs after to fold away the dead 463 // computations, and then ADCE will run later to exploit any new DCE 464 // opportunities that creates). 465 FPM.addPass(BDCEPass()); 466 467 // Run instcombine after redundancy and dead bit elimination to exploit 468 // opportunities opened up by them. 469 FPM.addPass(InstCombinePass()); 470 invokePeepholeEPCallbacks(FPM, Level); 471 472 // Re-consider control flow based optimizations after redundancy elimination, 473 // redo DCE, etc. 474 FPM.addPass(JumpThreadingPass()); 475 FPM.addPass(CorrelatedValuePropagationPass()); 476 FPM.addPass(DSEPass()); 477 FPM.addPass(createFunctionToLoopPassAdaptor(LICMPass(), DebugLogging)); 478 479 for (auto &C : ScalarOptimizerLateEPCallbacks) 480 C(FPM, Level); 481 482 // Finally, do an expensive DCE pass to catch all the dead code exposed by 483 // the simplifications and basic cleanup after all the simplifications. 484 FPM.addPass(ADCEPass()); 485 FPM.addPass(SimplifyCFGPass()); 486 FPM.addPass(InstCombinePass()); 487 invokePeepholeEPCallbacks(FPM, Level); 488 489 return FPM; 490 } 491 492 void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging, 493 PassBuilder::OptimizationLevel Level, 494 bool RunProfileGen, 495 std::string ProfileGenFile, 496 std::string ProfileUseFile) { 497 // Generally running simplification passes and the inliner with an high 498 // threshold results in smaller executables, but there may be cases where 499 // the size grows, so let's be conservative here and skip this simplification 500 // at -Os/Oz. 501 if (!isOptimizingForSize(Level)) { 502 InlineParams IP; 503 504 // In the old pass manager, this is a cl::opt. Should still this be one? 505 IP.DefaultThreshold = 75; 506 507 // FIXME: The hint threshold has the same value used by the regular inliner. 508 // This should probably be lowered after performance testing. 509 // FIXME: this comment is cargo culted from the old pass manager, revisit). 510 IP.HintThreshold = 325; 511 512 CGSCCPassManager CGPipeline(DebugLogging); 513 514 CGPipeline.addPass(InlinerPass(IP)); 515 516 FunctionPassManager FPM; 517 FPM.addPass(SROA()); 518 FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies. 519 FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks. 520 FPM.addPass(InstCombinePass()); // Combine silly sequences. 521 invokePeepholeEPCallbacks(FPM, Level); 522 523 CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); 524 525 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPipeline))); 526 } 527 528 // Delete anything that is now dead to make sure that we don't instrument 529 // dead code. Instrumentation can end up keeping dead code around and 530 // dramatically increase code size. 531 MPM.addPass(GlobalDCEPass()); 532 533 if (RunProfileGen) { 534 MPM.addPass(PGOInstrumentationGen()); 535 536 FunctionPassManager FPM; 537 FPM.addPass( 538 createFunctionToLoopPassAdaptor(LoopRotatePass(), DebugLogging)); 539 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 540 541 // Add the profile lowering pass. 542 InstrProfOptions Options; 543 if (!ProfileGenFile.empty()) 544 Options.InstrProfileOutput = ProfileGenFile; 545 Options.DoCounterPromotion = true; 546 MPM.addPass(InstrProfiling(Options)); 547 } 548 549 if (!ProfileUseFile.empty()) 550 MPM.addPass(PGOInstrumentationUse(ProfileUseFile)); 551 } 552 553 static InlineParams 554 getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) { 555 auto O3 = PassBuilder::O3; 556 unsigned OptLevel = Level > O3 ? 2 : Level; 557 unsigned SizeLevel = Level > O3 ? Level - O3 : 0; 558 return getInlineParams(OptLevel, SizeLevel); 559 } 560 561 ModulePassManager 562 PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, 563 ThinLTOPhase Phase, 564 bool DebugLogging) { 565 ModulePassManager MPM(DebugLogging); 566 567 // Do basic inference of function attributes from known properties of system 568 // libraries and other oracles. 569 MPM.addPass(InferFunctionAttrsPass()); 570 571 // Create an early function pass manager to cleanup the output of the 572 // frontend. 573 FunctionPassManager EarlyFPM(DebugLogging); 574 EarlyFPM.addPass(SimplifyCFGPass()); 575 EarlyFPM.addPass(SROA()); 576 EarlyFPM.addPass(EarlyCSEPass()); 577 EarlyFPM.addPass(LowerExpectIntrinsicPass()); 578 if (Level == O3) 579 EarlyFPM.addPass(CallSiteSplittingPass()); 580 581 // In SamplePGO ThinLTO backend, we need instcombine before profile annotation 582 // to convert bitcast to direct calls so that they can be inlined during the 583 // profile annotation prepration step. 584 // More details about SamplePGO design can be found in: 585 // https://research.google.com/pubs/pub45290.html 586 // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured. 587 if (PGOOpt && !PGOOpt->SampleProfileFile.empty() && 588 Phase == ThinLTOPhase::PostLink) 589 EarlyFPM.addPass(InstCombinePass()); 590 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM))); 591 592 if (PGOOpt && !PGOOpt->SampleProfileFile.empty()) { 593 // Annotate sample profile right after early FPM to ensure freshness of 594 // the debug info. 595 MPM.addPass(SampleProfileLoaderPass(PGOOpt->SampleProfileFile, 596 Phase == ThinLTOPhase::PreLink)); 597 // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard 598 // for the profile annotation to be accurate in the ThinLTO backend. 599 if (Phase != ThinLTOPhase::PreLink) 600 // We perform early indirect call promotion here, before globalopt. 601 // This is important for the ThinLTO backend phase because otherwise 602 // imported available_externally functions look unreferenced and are 603 // removed. 604 MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink, 605 true)); 606 } 607 608 // Interprocedural constant propagation now that basic cleanup has occurred 609 // and prior to optimizing globals. 610 // FIXME: This position in the pipeline hasn't been carefully considered in 611 // years, it should be re-analyzed. 612 MPM.addPass(IPSCCPPass()); 613 614 // Attach metadata to indirect call sites indicating the set of functions 615 // they may target at run-time. This should follow IPSCCP. 616 MPM.addPass(CalledValuePropagationPass()); 617 618 // Optimize globals to try and fold them into constants. 619 MPM.addPass(GlobalOptPass()); 620 621 // Promote any localized globals to SSA registers. 622 // FIXME: Should this instead by a run of SROA? 623 // FIXME: We should probably run instcombine and simplify-cfg afterward to 624 // delete control flows that are dead once globals have been folded to 625 // constants. 626 MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass())); 627 628 // Remove any dead arguments exposed by cleanups and constand folding 629 // globals. 630 MPM.addPass(DeadArgumentEliminationPass()); 631 632 // Create a small function pass pipeline to cleanup after all the global 633 // optimizations. 634 FunctionPassManager GlobalCleanupPM(DebugLogging); 635 GlobalCleanupPM.addPass(InstCombinePass()); 636 invokePeepholeEPCallbacks(GlobalCleanupPM, Level); 637 638 GlobalCleanupPM.addPass(SimplifyCFGPass()); 639 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM))); 640 641 // Add all the requested passes for instrumentation PGO, if requested. 642 if (PGOOpt && Phase != ThinLTOPhase::PostLink && 643 (!PGOOpt->ProfileGenFile.empty() || !PGOOpt->ProfileUseFile.empty())) { 644 addPGOInstrPasses(MPM, DebugLogging, Level, PGOOpt->RunProfileGen, 645 PGOOpt->ProfileGenFile, PGOOpt->ProfileUseFile); 646 MPM.addPass(PGOIndirectCallPromotion(false, false)); 647 } 648 649 // Synthesize function entry counts for non-PGO compilation. 650 if (EnableSyntheticCounts && !PGOOpt) 651 MPM.addPass(SyntheticCountsPropagation()); 652 653 // Require the GlobalsAA analysis for the module so we can query it within 654 // the CGSCC pipeline. 655 MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>()); 656 657 // Require the ProfileSummaryAnalysis for the module so we can query it within 658 // the inliner pass. 659 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>()); 660 661 // Now begin the main postorder CGSCC pipeline. 662 // FIXME: The current CGSCC pipeline has its origins in the legacy pass 663 // manager and trying to emulate its precise behavior. Much of this doesn't 664 // make a lot of sense and we should revisit the core CGSCC structure. 665 CGSCCPassManager MainCGPipeline(DebugLogging); 666 667 // Note: historically, the PruneEH pass was run first to deduce nounwind and 668 // generally clean up exception handling overhead. It isn't clear this is 669 // valuable as the inliner doesn't currently care whether it is inlining an 670 // invoke or a call. 671 672 // Run the inliner first. The theory is that we are walking bottom-up and so 673 // the callees have already been fully optimized, and we want to inline them 674 // into the callers so that our optimizations can reflect that. 675 // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO 676 // because it makes profile annotation in the backend inaccurate. 677 InlineParams IP = getInlineParamsFromOptLevel(Level); 678 if (Phase == ThinLTOPhase::PreLink && 679 PGOOpt && !PGOOpt->SampleProfileFile.empty()) 680 IP.HotCallSiteThreshold = 0; 681 MainCGPipeline.addPass(InlinerPass(IP)); 682 683 // Now deduce any function attributes based in the current code. 684 MainCGPipeline.addPass(PostOrderFunctionAttrsPass()); 685 686 // When at O3 add argument promotion to the pass pipeline. 687 // FIXME: It isn't at all clear why this should be limited to O3. 688 if (Level == O3) 689 MainCGPipeline.addPass(ArgumentPromotionPass()); 690 691 // Lastly, add the core function simplification pipeline nested inside the 692 // CGSCC walk. 693 MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor( 694 buildFunctionSimplificationPipeline(Level, Phase, DebugLogging))); 695 696 for (auto &C : CGSCCOptimizerLateEPCallbacks) 697 C(MainCGPipeline, Level); 698 699 // We wrap the CGSCC pipeline in a devirtualization repeater. This will try 700 // to detect when we devirtualize indirect calls and iterate the SCC passes 701 // in that case to try and catch knock-on inlining or function attrs 702 // opportunities. Then we add it to the module pipeline by walking the SCCs 703 // in postorder (or bottom-up). 704 MPM.addPass( 705 createModuleToPostOrderCGSCCPassAdaptor(createDevirtSCCRepeatedPass( 706 std::move(MainCGPipeline), MaxDevirtIterations))); 707 708 return MPM; 709 } 710 711 ModulePassManager 712 PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level, 713 bool DebugLogging) { 714 ModulePassManager MPM(DebugLogging); 715 716 // Optimize globals now that the module is fully simplified. 717 MPM.addPass(GlobalOptPass()); 718 MPM.addPass(GlobalDCEPass()); 719 720 // Run partial inlining pass to partially inline functions that have 721 // large bodies. 722 if (RunPartialInlining) 723 MPM.addPass(PartialInlinerPass()); 724 725 // Remove avail extern fns and globals definitions since we aren't compiling 726 // an object file for later LTO. For LTO we want to preserve these so they 727 // are eligible for inlining at link-time. Note if they are unreferenced they 728 // will be removed by GlobalDCE later, so this only impacts referenced 729 // available externally globals. Eventually they will be suppressed during 730 // codegen, but eliminating here enables more opportunity for GlobalDCE as it 731 // may make globals referenced by available external functions dead and saves 732 // running remaining passes on the eliminated functions. 733 MPM.addPass(EliminateAvailableExternallyPass()); 734 735 // Do RPO function attribute inference across the module to forward-propagate 736 // attributes where applicable. 737 // FIXME: Is this really an optimization rather than a canonicalization? 738 MPM.addPass(ReversePostOrderFunctionAttrsPass()); 739 740 // Re-require GloblasAA here prior to function passes. This is particularly 741 // useful as the above will have inlined, DCE'ed, and function-attr 742 // propagated everything. We should at this point have a reasonably minimal 743 // and richly annotated call graph. By computing aliasing and mod/ref 744 // information for all local globals here, the late loop passes and notably 745 // the vectorizer will be able to use them to help recognize vectorizable 746 // memory operations. 747 MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>()); 748 749 FunctionPassManager OptimizePM(DebugLogging); 750 OptimizePM.addPass(Float2IntPass()); 751 // FIXME: We need to run some loop optimizations to re-rotate loops after 752 // simplify-cfg and others undo their rotation. 753 754 // Optimize the loop execution. These passes operate on entire loop nests 755 // rather than on each loop in an inside-out manner, and so they are actually 756 // function passes. 757 758 for (auto &C : VectorizerStartEPCallbacks) 759 C(OptimizePM, Level); 760 761 // First rotate loops that may have been un-rotated by prior passes. 762 OptimizePM.addPass( 763 createFunctionToLoopPassAdaptor(LoopRotatePass(), DebugLogging)); 764 765 // Distribute loops to allow partial vectorization. I.e. isolate dependences 766 // into separate loop that would otherwise inhibit vectorization. This is 767 // currently only performed for loops marked with the metadata 768 // llvm.loop.distribute=true or when -enable-loop-distribute is specified. 769 OptimizePM.addPass(LoopDistributePass()); 770 771 // Now run the core loop vectorizer. 772 OptimizePM.addPass(LoopVectorizePass()); 773 774 // Eliminate loads by forwarding stores from the previous iteration to loads 775 // of the current iteration. 776 OptimizePM.addPass(LoopLoadEliminationPass()); 777 778 // Cleanup after the loop optimization passes. 779 OptimizePM.addPass(InstCombinePass()); 780 781 // Now that we've formed fast to execute loop structures, we do further 782 // optimizations. These are run afterward as they might block doing complex 783 // analyses and transforms such as what are needed for loop vectorization. 784 785 // Cleanup after loop vectorization, etc. Simplification passes like CVP and 786 // GVN, loop transforms, and others have already run, so it's now better to 787 // convert to more optimized IR using more aggressive simplify CFG options. 788 // The extra sinking transform can create larger basic blocks, so do this 789 // before SLP vectorization. 790 OptimizePM.addPass(SimplifyCFGPass(SimplifyCFGOptions(). 791 forwardSwitchCondToPhi(true). 792 convertSwitchToLookupTable(true). 793 needCanonicalLoops(false). 794 sinkCommonInsts(true))); 795 796 // Optimize parallel scalar instruction chains into SIMD instructions. 797 OptimizePM.addPass(SLPVectorizerPass()); 798 799 OptimizePM.addPass(InstCombinePass()); 800 801 // Unroll small loops to hide loop backedge latency and saturate any parallel 802 // execution resources of an out-of-order processor. We also then need to 803 // clean up redundancies and loop invariant code. 804 // FIXME: It would be really good to use a loop-integrated instruction 805 // combiner for cleanup here so that the unrolling and LICM can be pipelined 806 // across the loop nests. 807 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll 808 if (EnableUnrollAndJam) { 809 OptimizePM.addPass( 810 createFunctionToLoopPassAdaptor(LoopUnrollAndJamPass(Level))); 811 } 812 OptimizePM.addPass(LoopUnrollPass(Level)); 813 OptimizePM.addPass(InstCombinePass()); 814 OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>()); 815 OptimizePM.addPass(createFunctionToLoopPassAdaptor(LICMPass(), DebugLogging)); 816 817 // Now that we've vectorized and unrolled loops, we may have more refined 818 // alignment information, try to re-derive it here. 819 OptimizePM.addPass(AlignmentFromAssumptionsPass()); 820 821 // LoopSink pass sinks instructions hoisted by LICM, which serves as a 822 // canonicalization pass that enables other optimizations. As a result, 823 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM 824 // result too early. 825 OptimizePM.addPass(LoopSinkPass()); 826 827 // And finally clean up LCSSA form before generating code. 828 OptimizePM.addPass(InstSimplifyPass()); 829 830 // This hoists/decomposes div/rem ops. It should run after other sink/hoist 831 // passes to avoid re-sinking, but before SimplifyCFG because it can allow 832 // flattening of blocks. 833 OptimizePM.addPass(DivRemPairsPass()); 834 835 // LoopSink (and other loop passes since the last simplifyCFG) might have 836 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG. 837 OptimizePM.addPass(SimplifyCFGPass()); 838 839 // Optimize PHIs by speculating around them when profitable. Note that this 840 // pass needs to be run after any PRE or similar pass as it is essentially 841 // inserting redudnancies into the progrem. This even includes SimplifyCFG. 842 OptimizePM.addPass(SpeculateAroundPHIsPass()); 843 844 // Add the core optimizing pipeline. 845 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM))); 846 847 MPM.addPass(CGProfilePass()); 848 849 // Now we need to do some global optimization transforms. 850 // FIXME: It would seem like these should come first in the optimization 851 // pipeline and maybe be the bottom of the canonicalization pipeline? Weird 852 // ordering here. 853 MPM.addPass(GlobalDCEPass()); 854 MPM.addPass(ConstantMergePass()); 855 856 return MPM; 857 } 858 859 ModulePassManager 860 PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level, 861 bool DebugLogging) { 862 assert(Level != O0 && "Must request optimizations for the default pipeline!"); 863 864 ModulePassManager MPM(DebugLogging); 865 866 // Force any function attributes we want the rest of the pipeline to observe. 867 MPM.addPass(ForceFunctionAttrsPass()); 868 869 // Apply module pipeline start EP callback. 870 for (auto &C : PipelineStartEPCallbacks) 871 C(MPM); 872 873 if (PGOOpt && PGOOpt->SamplePGOSupport) 874 MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); 875 876 // Add the core simplification pipeline. 877 MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None, 878 DebugLogging)); 879 880 // Now add the optimization pipeline. 881 MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging)); 882 883 return MPM; 884 } 885 886 ModulePassManager 887 PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level, 888 bool DebugLogging) { 889 assert(Level != O0 && "Must request optimizations for the default pipeline!"); 890 891 ModulePassManager MPM(DebugLogging); 892 893 // Force any function attributes we want the rest of the pipeline to observe. 894 MPM.addPass(ForceFunctionAttrsPass()); 895 896 if (PGOOpt && PGOOpt->SamplePGOSupport) 897 MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); 898 899 // Apply module pipeline start EP callback. 900 for (auto &C : PipelineStartEPCallbacks) 901 C(MPM); 902 903 // If we are planning to perform ThinLTO later, we don't bloat the code with 904 // unrolling/vectorization/... now. Just simplify the module as much as we 905 // can. 906 MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink, 907 DebugLogging)); 908 909 // Run partial inlining pass to partially inline functions that have 910 // large bodies. 911 // FIXME: It isn't clear whether this is really the right place to run this 912 // in ThinLTO. Because there is another canonicalization and simplification 913 // phase that will run after the thin link, running this here ends up with 914 // less information than will be available later and it may grow functions in 915 // ways that aren't beneficial. 916 if (RunPartialInlining) 917 MPM.addPass(PartialInlinerPass()); 918 919 // Reduce the size of the IR as much as possible. 920 MPM.addPass(GlobalOptPass()); 921 922 return MPM; 923 } 924 925 ModulePassManager PassBuilder::buildThinLTODefaultPipeline( 926 OptimizationLevel Level, bool DebugLogging, 927 const ModuleSummaryIndex *ImportSummary) { 928 ModulePassManager MPM(DebugLogging); 929 930 if (ImportSummary) { 931 // These passes import type identifier resolutions for whole-program 932 // devirtualization and CFI. They must run early because other passes may 933 // disturb the specific instruction patterns that these passes look for, 934 // creating dependencies on resolutions that may not appear in the summary. 935 // 936 // For example, GVN may transform the pattern assume(type.test) appearing in 937 // two basic blocks into assume(phi(type.test, type.test)), which would 938 // transform a dependency on a WPD resolution into a dependency on a type 939 // identifier resolution for CFI. 940 // 941 // Also, WPD has access to more precise information than ICP and can 942 // devirtualize more effectively, so it should operate on the IR first. 943 MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary)); 944 MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary)); 945 } 946 947 // Force any function attributes we want the rest of the pipeline to observe. 948 MPM.addPass(ForceFunctionAttrsPass()); 949 950 // During the ThinLTO backend phase we perform early indirect call promotion 951 // here, before globalopt. Otherwise imported available_externally functions 952 // look unreferenced and are removed. 953 // FIXME: move this into buildModuleSimplificationPipeline to merge the logic 954 // with SamplePGO. 955 if (!PGOOpt || PGOOpt->SampleProfileFile.empty()) 956 MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, 957 false /* SamplePGO */)); 958 959 // Add the core simplification pipeline. 960 MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink, 961 DebugLogging)); 962 963 // Now add the optimization pipeline. 964 MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging)); 965 966 return MPM; 967 } 968 969 ModulePassManager 970 PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level, 971 bool DebugLogging) { 972 assert(Level != O0 && "Must request optimizations for the default pipeline!"); 973 // FIXME: We should use a customized pre-link pipeline! 974 return buildPerModuleDefaultPipeline(Level, DebugLogging); 975 } 976 977 ModulePassManager 978 PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging, 979 ModuleSummaryIndex *ExportSummary) { 980 assert(Level != O0 && "Must request optimizations for the default pipeline!"); 981 ModulePassManager MPM(DebugLogging); 982 983 // Remove unused virtual tables to improve the quality of code generated by 984 // whole-program devirtualization and bitset lowering. 985 MPM.addPass(GlobalDCEPass()); 986 987 // Force any function attributes we want the rest of the pipeline to observe. 988 MPM.addPass(ForceFunctionAttrsPass()); 989 990 // Do basic inference of function attributes from known properties of system 991 // libraries and other oracles. 992 MPM.addPass(InferFunctionAttrsPass()); 993 994 if (Level > 1) { 995 FunctionPassManager EarlyFPM(DebugLogging); 996 EarlyFPM.addPass(CallSiteSplittingPass()); 997 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM))); 998 999 // Indirect call promotion. This should promote all the targets that are 1000 // left by the earlier promotion pass that promotes intra-module targets. 1001 // This two-step promotion is to save the compile time. For LTO, it should 1002 // produce the same result as if we only do promotion here. 1003 MPM.addPass(PGOIndirectCallPromotion( 1004 true /* InLTO */, PGOOpt && !PGOOpt->SampleProfileFile.empty())); 1005 // Propagate constants at call sites into the functions they call. This 1006 // opens opportunities for globalopt (and inlining) by substituting function 1007 // pointers passed as arguments to direct uses of functions. 1008 MPM.addPass(IPSCCPPass()); 1009 1010 // Attach metadata to indirect call sites indicating the set of functions 1011 // they may target at run-time. This should follow IPSCCP. 1012 MPM.addPass(CalledValuePropagationPass()); 1013 } 1014 1015 // Now deduce any function attributes based in the current code. 1016 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( 1017 PostOrderFunctionAttrsPass())); 1018 1019 // Do RPO function attribute inference across the module to forward-propagate 1020 // attributes where applicable. 1021 // FIXME: Is this really an optimization rather than a canonicalization? 1022 MPM.addPass(ReversePostOrderFunctionAttrsPass()); 1023 1024 // Use inragne annotations on GEP indices to split globals where beneficial. 1025 MPM.addPass(GlobalSplitPass()); 1026 1027 // Run whole program optimization of virtual call when the list of callees 1028 // is fixed. 1029 MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr)); 1030 1031 // Stop here at -O1. 1032 if (Level == 1) { 1033 // The LowerTypeTestsPass needs to run to lower type metadata and the 1034 // type.test intrinsics. The pass does nothing if CFI is disabled. 1035 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr)); 1036 return MPM; 1037 } 1038 1039 // Optimize globals to try and fold them into constants. 1040 MPM.addPass(GlobalOptPass()); 1041 1042 // Promote any localized globals to SSA registers. 1043 MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass())); 1044 1045 // Linking modules together can lead to duplicate global constant, only 1046 // keep one copy of each constant. 1047 MPM.addPass(ConstantMergePass()); 1048 1049 // Remove unused arguments from functions. 1050 MPM.addPass(DeadArgumentEliminationPass()); 1051 1052 // Reduce the code after globalopt and ipsccp. Both can open up significant 1053 // simplification opportunities, and both can propagate functions through 1054 // function pointers. When this happens, we often have to resolve varargs 1055 // calls, etc, so let instcombine do this. 1056 FunctionPassManager PeepholeFPM(DebugLogging); 1057 if (Level == O3) 1058 PeepholeFPM.addPass(AggressiveInstCombinePass()); 1059 PeepholeFPM.addPass(InstCombinePass()); 1060 invokePeepholeEPCallbacks(PeepholeFPM, Level); 1061 1062 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM))); 1063 1064 // Note: historically, the PruneEH pass was run first to deduce nounwind and 1065 // generally clean up exception handling overhead. It isn't clear this is 1066 // valuable as the inliner doesn't currently care whether it is inlining an 1067 // invoke or a call. 1068 // Run the inliner now. 1069 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( 1070 InlinerPass(getInlineParamsFromOptLevel(Level)))); 1071 1072 // Optimize globals again after we ran the inliner. 1073 MPM.addPass(GlobalOptPass()); 1074 1075 // Garbage collect dead functions. 1076 // FIXME: Add ArgumentPromotion pass after once it's ported. 1077 MPM.addPass(GlobalDCEPass()); 1078 1079 FunctionPassManager FPM(DebugLogging); 1080 // The IPO Passes may leave cruft around. Clean up after them. 1081 FPM.addPass(InstCombinePass()); 1082 invokePeepholeEPCallbacks(FPM, Level); 1083 1084 FPM.addPass(JumpThreadingPass()); 1085 1086 // Break up allocas 1087 FPM.addPass(SROA()); 1088 1089 // Run a few AA driver optimizations here and now to cleanup the code. 1090 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 1091 1092 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( 1093 PostOrderFunctionAttrsPass())); 1094 // FIXME: here we run IP alias analysis in the legacy PM. 1095 1096 FunctionPassManager MainFPM; 1097 1098 // FIXME: once we fix LoopPass Manager, add LICM here. 1099 // FIXME: once we provide support for enabling MLSM, add it here. 1100 // FIXME: once we provide support for enabling NewGVN, add it here. 1101 if (RunNewGVN) 1102 MainFPM.addPass(NewGVNPass()); 1103 else 1104 MainFPM.addPass(GVN()); 1105 1106 // Remove dead memcpy()'s. 1107 MainFPM.addPass(MemCpyOptPass()); 1108 1109 // Nuke dead stores. 1110 MainFPM.addPass(DSEPass()); 1111 1112 // FIXME: at this point, we run a bunch of loop passes: 1113 // indVarSimplify, loopDeletion, loopInterchange, loopUnrool, 1114 // loopVectorize. Enable them once the remaining issue with LPM 1115 // are sorted out. 1116 1117 MainFPM.addPass(InstCombinePass()); 1118 MainFPM.addPass(SimplifyCFGPass()); 1119 MainFPM.addPass(SCCPPass()); 1120 MainFPM.addPass(InstCombinePass()); 1121 MainFPM.addPass(BDCEPass()); 1122 1123 // FIXME: We may want to run SLPVectorizer here. 1124 // After vectorization, assume intrinsics may tell us more 1125 // about pointer alignments. 1126 #if 0 1127 MainFPM.add(AlignmentFromAssumptionsPass()); 1128 #endif 1129 1130 // FIXME: Conditionally run LoadCombine here, after it's ported 1131 // (in case we still have this pass, given its questionable usefulness). 1132 1133 MainFPM.addPass(InstCombinePass()); 1134 invokePeepholeEPCallbacks(MainFPM, Level); 1135 MainFPM.addPass(JumpThreadingPass()); 1136 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM))); 1137 1138 // Create a function that performs CFI checks for cross-DSO calls with 1139 // targets in the current module. 1140 MPM.addPass(CrossDSOCFIPass()); 1141 1142 // Lower type metadata and the type.test intrinsic. This pass supports 1143 // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs 1144 // to be run at link time if CFI is enabled. This pass does nothing if 1145 // CFI is disabled. 1146 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr)); 1147 1148 // Add late LTO optimization passes. 1149 // Delete basic blocks, which optimization passes may have killed. 1150 MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass())); 1151 1152 // Drop bodies of available eternally objects to improve GlobalDCE. 1153 MPM.addPass(EliminateAvailableExternallyPass()); 1154 1155 // Now that we have optimized the program, discard unreachable functions. 1156 MPM.addPass(GlobalDCEPass()); 1157 1158 // FIXME: Enable MergeFuncs, conditionally, after ported, maybe. 1159 return MPM; 1160 } 1161 1162 AAManager PassBuilder::buildDefaultAAPipeline() { 1163 AAManager AA; 1164 1165 // The order in which these are registered determines their priority when 1166 // being queried. 1167 1168 // First we register the basic alias analysis that provides the majority of 1169 // per-function local AA logic. This is a stateless, on-demand local set of 1170 // AA techniques. 1171 AA.registerFunctionAnalysis<BasicAA>(); 1172 1173 // Next we query fast, specialized alias analyses that wrap IR-embedded 1174 // information about aliasing. 1175 AA.registerFunctionAnalysis<ScopedNoAliasAA>(); 1176 AA.registerFunctionAnalysis<TypeBasedAA>(); 1177 1178 // Add support for querying global aliasing information when available. 1179 // Because the `AAManager` is a function analysis and `GlobalsAA` is a module 1180 // analysis, all that the `AAManager` can do is query for any *cached* 1181 // results from `GlobalsAA` through a readonly proxy. 1182 AA.registerModuleAnalysis<GlobalsAA>(); 1183 1184 return AA; 1185 } 1186 1187 static Optional<int> parseRepeatPassName(StringRef Name) { 1188 if (!Name.consume_front("repeat<") || !Name.consume_back(">")) 1189 return None; 1190 int Count; 1191 if (Name.getAsInteger(0, Count) || Count <= 0) 1192 return None; 1193 return Count; 1194 } 1195 1196 static Optional<int> parseDevirtPassName(StringRef Name) { 1197 if (!Name.consume_front("devirt<") || !Name.consume_back(">")) 1198 return None; 1199 int Count; 1200 if (Name.getAsInteger(0, Count) || Count <= 0) 1201 return None; 1202 return Count; 1203 } 1204 1205 /// Tests whether a pass name starts with a valid prefix for a default pipeline 1206 /// alias. 1207 static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) { 1208 return Name.startswith("default") || Name.startswith("thinlto") || 1209 Name.startswith("lto"); 1210 } 1211 1212 /// Tests whether registered callbacks will accept a given pass name. 1213 /// 1214 /// When parsing a pipeline text, the type of the outermost pipeline may be 1215 /// omitted, in which case the type is automatically determined from the first 1216 /// pass name in the text. This may be a name that is handled through one of the 1217 /// callbacks. We check this through the oridinary parsing callbacks by setting 1218 /// up a dummy PassManager in order to not force the client to also handle this 1219 /// type of query. 1220 template <typename PassManagerT, typename CallbacksT> 1221 static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) { 1222 if (!Callbacks.empty()) { 1223 PassManagerT DummyPM; 1224 for (auto &CB : Callbacks) 1225 if (CB(Name, DummyPM, {})) 1226 return true; 1227 } 1228 return false; 1229 } 1230 1231 template <typename CallbacksT> 1232 static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) { 1233 // Manually handle aliases for pre-configured pipeline fragments. 1234 if (startsWithDefaultPipelineAliasPrefix(Name)) 1235 return DefaultAliasRegex.match(Name); 1236 1237 // Explicitly handle pass manager names. 1238 if (Name == "module") 1239 return true; 1240 if (Name == "cgscc") 1241 return true; 1242 if (Name == "function") 1243 return true; 1244 1245 // Explicitly handle custom-parsed pass names. 1246 if (parseRepeatPassName(Name)) 1247 return true; 1248 1249 #define MODULE_PASS(NAME, CREATE_PASS) \ 1250 if (Name == NAME) \ 1251 return true; 1252 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \ 1253 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ 1254 return true; 1255 #include "PassRegistry.def" 1256 1257 return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks); 1258 } 1259 1260 template <typename CallbacksT> 1261 static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) { 1262 // Explicitly handle pass manager names. 1263 if (Name == "cgscc") 1264 return true; 1265 if (Name == "function") 1266 return true; 1267 1268 // Explicitly handle custom-parsed pass names. 1269 if (parseRepeatPassName(Name)) 1270 return true; 1271 if (parseDevirtPassName(Name)) 1272 return true; 1273 1274 #define CGSCC_PASS(NAME, CREATE_PASS) \ 1275 if (Name == NAME) \ 1276 return true; 1277 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \ 1278 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ 1279 return true; 1280 #include "PassRegistry.def" 1281 1282 return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks); 1283 } 1284 1285 template <typename CallbacksT> 1286 static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) { 1287 // Explicitly handle pass manager names. 1288 if (Name == "function") 1289 return true; 1290 if (Name == "loop") 1291 return true; 1292 1293 // Explicitly handle custom-parsed pass names. 1294 if (parseRepeatPassName(Name)) 1295 return true; 1296 1297 #define FUNCTION_PASS(NAME, CREATE_PASS) \ 1298 if (Name == NAME) \ 1299 return true; 1300 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ 1301 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ 1302 return true; 1303 #include "PassRegistry.def" 1304 1305 return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks); 1306 } 1307 1308 template <typename CallbacksT> 1309 static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) { 1310 // Explicitly handle pass manager names. 1311 if (Name == "loop") 1312 return true; 1313 1314 // Explicitly handle custom-parsed pass names. 1315 if (parseRepeatPassName(Name)) 1316 return true; 1317 1318 #define LOOP_PASS(NAME, CREATE_PASS) \ 1319 if (Name == NAME) \ 1320 return true; 1321 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \ 1322 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \ 1323 return true; 1324 #include "PassRegistry.def" 1325 1326 return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks); 1327 } 1328 1329 Optional<std::vector<PassBuilder::PipelineElement>> 1330 PassBuilder::parsePipelineText(StringRef Text) { 1331 std::vector<PipelineElement> ResultPipeline; 1332 1333 SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = { 1334 &ResultPipeline}; 1335 for (;;) { 1336 std::vector<PipelineElement> &Pipeline = *PipelineStack.back(); 1337 size_t Pos = Text.find_first_of(",()"); 1338 Pipeline.push_back({Text.substr(0, Pos), {}}); 1339 1340 // If we have a single terminating name, we're done. 1341 if (Pos == Text.npos) 1342 break; 1343 1344 char Sep = Text[Pos]; 1345 Text = Text.substr(Pos + 1); 1346 if (Sep == ',') 1347 // Just a name ending in a comma, continue. 1348 continue; 1349 1350 if (Sep == '(') { 1351 // Push the inner pipeline onto the stack to continue processing. 1352 PipelineStack.push_back(&Pipeline.back().InnerPipeline); 1353 continue; 1354 } 1355 1356 assert(Sep == ')' && "Bogus separator!"); 1357 // When handling the close parenthesis, we greedily consume them to avoid 1358 // empty strings in the pipeline. 1359 do { 1360 // If we try to pop the outer pipeline we have unbalanced parentheses. 1361 if (PipelineStack.size() == 1) 1362 return None; 1363 1364 PipelineStack.pop_back(); 1365 } while (Text.consume_front(")")); 1366 1367 // Check if we've finished parsing. 1368 if (Text.empty()) 1369 break; 1370 1371 // Otherwise, the end of an inner pipeline always has to be followed by 1372 // a comma, and then we can continue. 1373 if (!Text.consume_front(",")) 1374 return None; 1375 } 1376 1377 if (PipelineStack.size() > 1) 1378 // Unbalanced paretheses. 1379 return None; 1380 1381 assert(PipelineStack.back() == &ResultPipeline && 1382 "Wrong pipeline at the bottom of the stack!"); 1383 return {std::move(ResultPipeline)}; 1384 } 1385 1386 bool PassBuilder::parseModulePass(ModulePassManager &MPM, 1387 const PipelineElement &E, bool VerifyEachPass, 1388 bool DebugLogging) { 1389 auto &Name = E.Name; 1390 auto &InnerPipeline = E.InnerPipeline; 1391 1392 // First handle complex passes like the pass managers which carry pipelines. 1393 if (!InnerPipeline.empty()) { 1394 if (Name == "module") { 1395 ModulePassManager NestedMPM(DebugLogging); 1396 if (!parseModulePassPipeline(NestedMPM, InnerPipeline, VerifyEachPass, 1397 DebugLogging)) 1398 return false; 1399 MPM.addPass(std::move(NestedMPM)); 1400 return true; 1401 } 1402 if (Name == "cgscc") { 1403 CGSCCPassManager CGPM(DebugLogging); 1404 if (!parseCGSCCPassPipeline(CGPM, InnerPipeline, VerifyEachPass, 1405 DebugLogging)) 1406 return false; 1407 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); 1408 return true; 1409 } 1410 if (Name == "function") { 1411 FunctionPassManager FPM(DebugLogging); 1412 if (!parseFunctionPassPipeline(FPM, InnerPipeline, VerifyEachPass, 1413 DebugLogging)) 1414 return false; 1415 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 1416 return true; 1417 } 1418 if (auto Count = parseRepeatPassName(Name)) { 1419 ModulePassManager NestedMPM(DebugLogging); 1420 if (!parseModulePassPipeline(NestedMPM, InnerPipeline, VerifyEachPass, 1421 DebugLogging)) 1422 return false; 1423 MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM))); 1424 return true; 1425 } 1426 1427 for (auto &C : ModulePipelineParsingCallbacks) 1428 if (C(Name, MPM, InnerPipeline)) 1429 return true; 1430 1431 // Normal passes can't have pipelines. 1432 return false; 1433 } 1434 1435 // Manually handle aliases for pre-configured pipeline fragments. 1436 if (startsWithDefaultPipelineAliasPrefix(Name)) { 1437 SmallVector<StringRef, 3> Matches; 1438 if (!DefaultAliasRegex.match(Name, &Matches)) 1439 return false; 1440 assert(Matches.size() == 3 && "Must capture two matched strings!"); 1441 1442 OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2]) 1443 .Case("O0", O0) 1444 .Case("O1", O1) 1445 .Case("O2", O2) 1446 .Case("O3", O3) 1447 .Case("Os", Os) 1448 .Case("Oz", Oz); 1449 if (L == O0) 1450 // At O0 we do nothing at all! 1451 return true; 1452 1453 if (Matches[1] == "default") { 1454 MPM.addPass(buildPerModuleDefaultPipeline(L, DebugLogging)); 1455 } else if (Matches[1] == "thinlto-pre-link") { 1456 MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L, DebugLogging)); 1457 } else if (Matches[1] == "thinlto") { 1458 MPM.addPass(buildThinLTODefaultPipeline(L, DebugLogging, nullptr)); 1459 } else if (Matches[1] == "lto-pre-link") { 1460 MPM.addPass(buildLTOPreLinkDefaultPipeline(L, DebugLogging)); 1461 } else { 1462 assert(Matches[1] == "lto" && "Not one of the matched options!"); 1463 MPM.addPass(buildLTODefaultPipeline(L, DebugLogging, nullptr)); 1464 } 1465 return true; 1466 } 1467 1468 // Finally expand the basic registered passes from the .inc file. 1469 #define MODULE_PASS(NAME, CREATE_PASS) \ 1470 if (Name == NAME) { \ 1471 MPM.addPass(CREATE_PASS); \ 1472 return true; \ 1473 } 1474 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \ 1475 if (Name == "require<" NAME ">") { \ 1476 MPM.addPass( \ 1477 RequireAnalysisPass< \ 1478 std::remove_reference<decltype(CREATE_PASS)>::type, Module>()); \ 1479 return true; \ 1480 } \ 1481 if (Name == "invalidate<" NAME ">") { \ 1482 MPM.addPass(InvalidateAnalysisPass< \ 1483 std::remove_reference<decltype(CREATE_PASS)>::type>()); \ 1484 return true; \ 1485 } 1486 #include "PassRegistry.def" 1487 1488 for (auto &C : ModulePipelineParsingCallbacks) 1489 if (C(Name, MPM, InnerPipeline)) 1490 return true; 1491 return false; 1492 } 1493 1494 bool PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM, 1495 const PipelineElement &E, bool VerifyEachPass, 1496 bool DebugLogging) { 1497 auto &Name = E.Name; 1498 auto &InnerPipeline = E.InnerPipeline; 1499 1500 // First handle complex passes like the pass managers which carry pipelines. 1501 if (!InnerPipeline.empty()) { 1502 if (Name == "cgscc") { 1503 CGSCCPassManager NestedCGPM(DebugLogging); 1504 if (!parseCGSCCPassPipeline(NestedCGPM, InnerPipeline, VerifyEachPass, 1505 DebugLogging)) 1506 return false; 1507 // Add the nested pass manager with the appropriate adaptor. 1508 CGPM.addPass(std::move(NestedCGPM)); 1509 return true; 1510 } 1511 if (Name == "function") { 1512 FunctionPassManager FPM(DebugLogging); 1513 if (!parseFunctionPassPipeline(FPM, InnerPipeline, VerifyEachPass, 1514 DebugLogging)) 1515 return false; 1516 // Add the nested pass manager with the appropriate adaptor. 1517 CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); 1518 return true; 1519 } 1520 if (auto Count = parseRepeatPassName(Name)) { 1521 CGSCCPassManager NestedCGPM(DebugLogging); 1522 if (!parseCGSCCPassPipeline(NestedCGPM, InnerPipeline, VerifyEachPass, 1523 DebugLogging)) 1524 return false; 1525 CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM))); 1526 return true; 1527 } 1528 if (auto MaxRepetitions = parseDevirtPassName(Name)) { 1529 CGSCCPassManager NestedCGPM(DebugLogging); 1530 if (!parseCGSCCPassPipeline(NestedCGPM, InnerPipeline, VerifyEachPass, 1531 DebugLogging)) 1532 return false; 1533 CGPM.addPass( 1534 createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions)); 1535 return true; 1536 } 1537 1538 for (auto &C : CGSCCPipelineParsingCallbacks) 1539 if (C(Name, CGPM, InnerPipeline)) 1540 return true; 1541 1542 // Normal passes can't have pipelines. 1543 return false; 1544 } 1545 1546 // Now expand the basic registered passes from the .inc file. 1547 #define CGSCC_PASS(NAME, CREATE_PASS) \ 1548 if (Name == NAME) { \ 1549 CGPM.addPass(CREATE_PASS); \ 1550 return true; \ 1551 } 1552 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \ 1553 if (Name == "require<" NAME ">") { \ 1554 CGPM.addPass(RequireAnalysisPass< \ 1555 std::remove_reference<decltype(CREATE_PASS)>::type, \ 1556 LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, \ 1557 CGSCCUpdateResult &>()); \ 1558 return true; \ 1559 } \ 1560 if (Name == "invalidate<" NAME ">") { \ 1561 CGPM.addPass(InvalidateAnalysisPass< \ 1562 std::remove_reference<decltype(CREATE_PASS)>::type>()); \ 1563 return true; \ 1564 } 1565 #include "PassRegistry.def" 1566 1567 for (auto &C : CGSCCPipelineParsingCallbacks) 1568 if (C(Name, CGPM, InnerPipeline)) 1569 return true; 1570 return false; 1571 } 1572 1573 bool PassBuilder::parseFunctionPass(FunctionPassManager &FPM, 1574 const PipelineElement &E, 1575 bool VerifyEachPass, bool DebugLogging) { 1576 auto &Name = E.Name; 1577 auto &InnerPipeline = E.InnerPipeline; 1578 1579 // First handle complex passes like the pass managers which carry pipelines. 1580 if (!InnerPipeline.empty()) { 1581 if (Name == "function") { 1582 FunctionPassManager NestedFPM(DebugLogging); 1583 if (!parseFunctionPassPipeline(NestedFPM, InnerPipeline, VerifyEachPass, 1584 DebugLogging)) 1585 return false; 1586 // Add the nested pass manager with the appropriate adaptor. 1587 FPM.addPass(std::move(NestedFPM)); 1588 return true; 1589 } 1590 if (Name == "loop") { 1591 LoopPassManager LPM(DebugLogging); 1592 if (!parseLoopPassPipeline(LPM, InnerPipeline, VerifyEachPass, 1593 DebugLogging)) 1594 return false; 1595 // Add the nested pass manager with the appropriate adaptor. 1596 FPM.addPass( 1597 createFunctionToLoopPassAdaptor(std::move(LPM), DebugLogging)); 1598 return true; 1599 } 1600 if (auto Count = parseRepeatPassName(Name)) { 1601 FunctionPassManager NestedFPM(DebugLogging); 1602 if (!parseFunctionPassPipeline(NestedFPM, InnerPipeline, VerifyEachPass, 1603 DebugLogging)) 1604 return false; 1605 FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM))); 1606 return true; 1607 } 1608 1609 for (auto &C : FunctionPipelineParsingCallbacks) 1610 if (C(Name, FPM, InnerPipeline)) 1611 return true; 1612 1613 // Normal passes can't have pipelines. 1614 return false; 1615 } 1616 1617 // Now expand the basic registered passes from the .inc file. 1618 #define FUNCTION_PASS(NAME, CREATE_PASS) \ 1619 if (Name == NAME) { \ 1620 FPM.addPass(CREATE_PASS); \ 1621 return true; \ 1622 } 1623 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ 1624 if (Name == "require<" NAME ">") { \ 1625 FPM.addPass( \ 1626 RequireAnalysisPass< \ 1627 std::remove_reference<decltype(CREATE_PASS)>::type, Function>()); \ 1628 return true; \ 1629 } \ 1630 if (Name == "invalidate<" NAME ">") { \ 1631 FPM.addPass(InvalidateAnalysisPass< \ 1632 std::remove_reference<decltype(CREATE_PASS)>::type>()); \ 1633 return true; \ 1634 } 1635 #include "PassRegistry.def" 1636 1637 for (auto &C : FunctionPipelineParsingCallbacks) 1638 if (C(Name, FPM, InnerPipeline)) 1639 return true; 1640 return false; 1641 } 1642 1643 bool PassBuilder::parseLoopPass(LoopPassManager &LPM, const PipelineElement &E, 1644 bool VerifyEachPass, bool DebugLogging) { 1645 StringRef Name = E.Name; 1646 auto &InnerPipeline = E.InnerPipeline; 1647 1648 // First handle complex passes like the pass managers which carry pipelines. 1649 if (!InnerPipeline.empty()) { 1650 if (Name == "loop") { 1651 LoopPassManager NestedLPM(DebugLogging); 1652 if (!parseLoopPassPipeline(NestedLPM, InnerPipeline, VerifyEachPass, 1653 DebugLogging)) 1654 return false; 1655 // Add the nested pass manager with the appropriate adaptor. 1656 LPM.addPass(std::move(NestedLPM)); 1657 return true; 1658 } 1659 if (auto Count = parseRepeatPassName(Name)) { 1660 LoopPassManager NestedLPM(DebugLogging); 1661 if (!parseLoopPassPipeline(NestedLPM, InnerPipeline, VerifyEachPass, 1662 DebugLogging)) 1663 return false; 1664 LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM))); 1665 return true; 1666 } 1667 1668 for (auto &C : LoopPipelineParsingCallbacks) 1669 if (C(Name, LPM, InnerPipeline)) 1670 return true; 1671 1672 // Normal passes can't have pipelines. 1673 return false; 1674 } 1675 1676 // Now expand the basic registered passes from the .inc file. 1677 #define LOOP_PASS(NAME, CREATE_PASS) \ 1678 if (Name == NAME) { \ 1679 LPM.addPass(CREATE_PASS); \ 1680 return true; \ 1681 } 1682 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \ 1683 if (Name == "require<" NAME ">") { \ 1684 LPM.addPass(RequireAnalysisPass< \ 1685 std::remove_reference<decltype(CREATE_PASS)>::type, Loop, \ 1686 LoopAnalysisManager, LoopStandardAnalysisResults &, \ 1687 LPMUpdater &>()); \ 1688 return true; \ 1689 } \ 1690 if (Name == "invalidate<" NAME ">") { \ 1691 LPM.addPass(InvalidateAnalysisPass< \ 1692 std::remove_reference<decltype(CREATE_PASS)>::type>()); \ 1693 return true; \ 1694 } 1695 #include "PassRegistry.def" 1696 1697 for (auto &C : LoopPipelineParsingCallbacks) 1698 if (C(Name, LPM, InnerPipeline)) 1699 return true; 1700 return false; 1701 } 1702 1703 bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) { 1704 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ 1705 if (Name == NAME) { \ 1706 AA.registerModuleAnalysis< \ 1707 std::remove_reference<decltype(CREATE_PASS)>::type>(); \ 1708 return true; \ 1709 } 1710 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ 1711 if (Name == NAME) { \ 1712 AA.registerFunctionAnalysis< \ 1713 std::remove_reference<decltype(CREATE_PASS)>::type>(); \ 1714 return true; \ 1715 } 1716 #include "PassRegistry.def" 1717 1718 for (auto &C : AAParsingCallbacks) 1719 if (C(Name, AA)) 1720 return true; 1721 return false; 1722 } 1723 1724 bool PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM, 1725 ArrayRef<PipelineElement> Pipeline, 1726 bool VerifyEachPass, 1727 bool DebugLogging) { 1728 for (const auto &Element : Pipeline) { 1729 if (!parseLoopPass(LPM, Element, VerifyEachPass, DebugLogging)) 1730 return false; 1731 // FIXME: No verifier support for Loop passes! 1732 } 1733 return true; 1734 } 1735 1736 bool PassBuilder::parseFunctionPassPipeline(FunctionPassManager &FPM, 1737 ArrayRef<PipelineElement> Pipeline, 1738 bool VerifyEachPass, 1739 bool DebugLogging) { 1740 for (const auto &Element : Pipeline) { 1741 if (!parseFunctionPass(FPM, Element, VerifyEachPass, DebugLogging)) 1742 return false; 1743 if (VerifyEachPass) 1744 FPM.addPass(VerifierPass()); 1745 } 1746 return true; 1747 } 1748 1749 bool PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM, 1750 ArrayRef<PipelineElement> Pipeline, 1751 bool VerifyEachPass, 1752 bool DebugLogging) { 1753 for (const auto &Element : Pipeline) { 1754 if (!parseCGSCCPass(CGPM, Element, VerifyEachPass, DebugLogging)) 1755 return false; 1756 // FIXME: No verifier support for CGSCC passes! 1757 } 1758 return true; 1759 } 1760 1761 void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM, 1762 FunctionAnalysisManager &FAM, 1763 CGSCCAnalysisManager &CGAM, 1764 ModuleAnalysisManager &MAM) { 1765 MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); }); 1766 MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); }); 1767 CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); }); 1768 FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); }); 1769 FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); }); 1770 FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); }); 1771 LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); }); 1772 } 1773 1774 bool PassBuilder::parseModulePassPipeline(ModulePassManager &MPM, 1775 ArrayRef<PipelineElement> Pipeline, 1776 bool VerifyEachPass, 1777 bool DebugLogging) { 1778 for (const auto &Element : Pipeline) { 1779 if (!parseModulePass(MPM, Element, VerifyEachPass, DebugLogging)) 1780 return false; 1781 if (VerifyEachPass) 1782 MPM.addPass(VerifierPass()); 1783 } 1784 return true; 1785 } 1786 1787 // Primary pass pipeline description parsing routine for a \c ModulePassManager 1788 // FIXME: Should this routine accept a TargetMachine or require the caller to 1789 // pre-populate the analysis managers with target-specific stuff? 1790 bool PassBuilder::parsePassPipeline(ModulePassManager &MPM, 1791 StringRef PipelineText, bool VerifyEachPass, 1792 bool DebugLogging) { 1793 auto Pipeline = parsePipelineText(PipelineText); 1794 if (!Pipeline || Pipeline->empty()) 1795 return false; 1796 1797 // If the first name isn't at the module layer, wrap the pipeline up 1798 // automatically. 1799 StringRef FirstName = Pipeline->front().Name; 1800 1801 if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) { 1802 if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) { 1803 Pipeline = {{"cgscc", std::move(*Pipeline)}}; 1804 } else if (isFunctionPassName(FirstName, 1805 FunctionPipelineParsingCallbacks)) { 1806 Pipeline = {{"function", std::move(*Pipeline)}}; 1807 } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) { 1808 Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}}; 1809 } else { 1810 for (auto &C : TopLevelPipelineParsingCallbacks) 1811 if (C(MPM, *Pipeline, VerifyEachPass, DebugLogging)) 1812 return true; 1813 1814 // Unknown pass name! 1815 return false; 1816 } 1817 } 1818 1819 return parseModulePassPipeline(MPM, *Pipeline, VerifyEachPass, DebugLogging); 1820 } 1821 1822 // Primary pass pipeline description parsing routine for a \c CGSCCPassManager 1823 bool PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM, 1824 StringRef PipelineText, bool VerifyEachPass, 1825 bool DebugLogging) { 1826 auto Pipeline = parsePipelineText(PipelineText); 1827 if (!Pipeline || Pipeline->empty()) 1828 return false; 1829 1830 StringRef FirstName = Pipeline->front().Name; 1831 if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) 1832 return false; 1833 1834 return parseCGSCCPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging); 1835 } 1836 1837 // Primary pass pipeline description parsing routine for a \c 1838 // FunctionPassManager 1839 bool PassBuilder::parsePassPipeline(FunctionPassManager &FPM, 1840 StringRef PipelineText, bool VerifyEachPass, 1841 bool DebugLogging) { 1842 auto Pipeline = parsePipelineText(PipelineText); 1843 if (!Pipeline || Pipeline->empty()) 1844 return false; 1845 1846 StringRef FirstName = Pipeline->front().Name; 1847 if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks)) 1848 return false; 1849 1850 return parseFunctionPassPipeline(FPM, *Pipeline, VerifyEachPass, 1851 DebugLogging); 1852 } 1853 1854 // Primary pass pipeline description parsing routine for a \c LoopPassManager 1855 bool PassBuilder::parsePassPipeline(LoopPassManager &CGPM, 1856 StringRef PipelineText, bool VerifyEachPass, 1857 bool DebugLogging) { 1858 auto Pipeline = parsePipelineText(PipelineText); 1859 if (!Pipeline || Pipeline->empty()) 1860 return false; 1861 1862 return parseLoopPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging); 1863 } 1864 1865 bool PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) { 1866 // If the pipeline just consists of the word 'default' just replace the AA 1867 // manager with our default one. 1868 if (PipelineText == "default") { 1869 AA = buildDefaultAAPipeline(); 1870 return true; 1871 } 1872 1873 while (!PipelineText.empty()) { 1874 StringRef Name; 1875 std::tie(Name, PipelineText) = PipelineText.split(','); 1876 if (!parseAAPassName(AA, Name)) 1877 return false; 1878 } 1879 1880 return true; 1881 } 1882