1 //===--- Tooling.cpp - Running clang standalone tools ---------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file implements functions to run clang tools standalone instead 11 // of running them as a plugin. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "clang/Tooling/Tooling.h" 16 #include "clang/AST/ASTConsumer.h" 17 #include "clang/Driver/Compilation.h" 18 #include "clang/Driver/Driver.h" 19 #include "clang/Driver/Tool.h" 20 #include "clang/Driver/ToolChain.h" 21 #include "clang/Frontend/ASTUnit.h" 22 #include "clang/Frontend/CompilerInstance.h" 23 #include "clang/Frontend/FrontendDiagnostic.h" 24 #include "clang/Frontend/TextDiagnosticPrinter.h" 25 #include "clang/Tooling/ArgumentsAdjusters.h" 26 #include "clang/Tooling/CompilationDatabase.h" 27 #include "llvm/ADT/STLExtras.h" 28 #include "llvm/Config/llvm-config.h" 29 #include "llvm/Option/Option.h" 30 #include "llvm/Support/Debug.h" 31 #include "llvm/Support/FileSystem.h" 32 #include "llvm/Support/Host.h" 33 #include "llvm/Support/raw_ostream.h" 34 35 #define DEBUG_TYPE "clang-tooling" 36 37 namespace clang { 38 namespace tooling { 39 40 ToolAction::~ToolAction() {} 41 42 FrontendActionFactory::~FrontendActionFactory() {} 43 44 // FIXME: This file contains structural duplication with other parts of the 45 // code that sets up a compiler to run tools on it, and we should refactor 46 // it to be based on the same framework. 47 48 /// \brief Builds a clang driver initialized for running clang tools. 49 static clang::driver::Driver *newDriver( 50 clang::DiagnosticsEngine *Diagnostics, const char *BinaryName, 51 IntrusiveRefCntPtr<vfs::FileSystem> VFS) { 52 clang::driver::Driver *CompilerDriver = new clang::driver::Driver( 53 BinaryName, llvm::sys::getDefaultTargetTriple(), *Diagnostics, VFS); 54 CompilerDriver->setTitle("clang_based_tool"); 55 return CompilerDriver; 56 } 57 58 /// \brief Retrieves the clang CC1 specific flags out of the compilation's jobs. 59 /// 60 /// Returns NULL on error. 61 static const llvm::opt::ArgStringList *getCC1Arguments( 62 clang::DiagnosticsEngine *Diagnostics, 63 clang::driver::Compilation *Compilation) { 64 // We expect to get back exactly one Command job, if we didn't something 65 // failed. Extract that job from the Compilation. 66 const clang::driver::JobList &Jobs = Compilation->getJobs(); 67 if (Jobs.size() != 1 || !isa<clang::driver::Command>(*Jobs.begin())) { 68 SmallString<256> error_msg; 69 llvm::raw_svector_ostream error_stream(error_msg); 70 Jobs.Print(error_stream, "; ", true); 71 Diagnostics->Report(clang::diag::err_fe_expected_compiler_job) 72 << error_stream.str(); 73 return nullptr; 74 } 75 76 // The one job we find should be to invoke clang again. 77 const clang::driver::Command &Cmd = 78 cast<clang::driver::Command>(*Jobs.begin()); 79 if (StringRef(Cmd.getCreator().getName()) != "clang") { 80 Diagnostics->Report(clang::diag::err_fe_expected_clang_command); 81 return nullptr; 82 } 83 84 return &Cmd.getArguments(); 85 } 86 87 /// \brief Returns a clang build invocation initialized from the CC1 flags. 88 clang::CompilerInvocation *newInvocation( 89 clang::DiagnosticsEngine *Diagnostics, 90 const llvm::opt::ArgStringList &CC1Args) { 91 assert(!CC1Args.empty() && "Must at least contain the program name!"); 92 clang::CompilerInvocation *Invocation = new clang::CompilerInvocation; 93 clang::CompilerInvocation::CreateFromArgs( 94 *Invocation, CC1Args.data() + 1, CC1Args.data() + CC1Args.size(), 95 *Diagnostics); 96 Invocation->getFrontendOpts().DisableFree = false; 97 Invocation->getCodeGenOpts().DisableFree = false; 98 Invocation->getDependencyOutputOpts() = DependencyOutputOptions(); 99 return Invocation; 100 } 101 102 bool runToolOnCode(clang::FrontendAction *ToolAction, const Twine &Code, 103 const Twine &FileName, 104 std::shared_ptr<PCHContainerOperations> PCHContainerOps) { 105 return runToolOnCodeWithArgs(ToolAction, Code, std::vector<std::string>(), 106 FileName, PCHContainerOps); 107 } 108 109 static std::vector<std::string> 110 getSyntaxOnlyToolArgs(const std::vector<std::string> &ExtraArgs, 111 StringRef FileName) { 112 std::vector<std::string> Args; 113 Args.push_back("clang-tool"); 114 Args.push_back("-fsyntax-only"); 115 Args.insert(Args.end(), ExtraArgs.begin(), ExtraArgs.end()); 116 Args.push_back(FileName.str()); 117 return Args; 118 } 119 120 bool runToolOnCodeWithArgs( 121 clang::FrontendAction *ToolAction, const Twine &Code, 122 const std::vector<std::string> &Args, const Twine &FileName, 123 std::shared_ptr<PCHContainerOperations> PCHContainerOps, 124 const FileContentMappings &VirtualMappedFiles) { 125 126 SmallString<16> FileNameStorage; 127 StringRef FileNameRef = FileName.toNullTerminatedStringRef(FileNameStorage); 128 llvm::IntrusiveRefCntPtr<vfs::OverlayFileSystem> OverlayFileSystem( 129 new vfs::OverlayFileSystem(vfs::getRealFileSystem())); 130 llvm::IntrusiveRefCntPtr<vfs::InMemoryFileSystem> InMemoryFileSystem( 131 new vfs::InMemoryFileSystem); 132 OverlayFileSystem->pushOverlay(InMemoryFileSystem); 133 llvm::IntrusiveRefCntPtr<FileManager> Files( 134 new FileManager(FileSystemOptions(), OverlayFileSystem)); 135 ToolInvocation Invocation(getSyntaxOnlyToolArgs(Args, FileNameRef), 136 ToolAction, Files.get(), PCHContainerOps); 137 138 SmallString<1024> CodeStorage; 139 InMemoryFileSystem->addFile(FileNameRef, 0, 140 llvm::MemoryBuffer::getMemBuffer( 141 Code.toNullTerminatedStringRef(CodeStorage))); 142 143 for (auto &FilenameWithContent : VirtualMappedFiles) { 144 InMemoryFileSystem->addFile( 145 FilenameWithContent.first, 0, 146 llvm::MemoryBuffer::getMemBuffer(FilenameWithContent.second)); 147 } 148 149 return Invocation.run(); 150 } 151 152 std::string getAbsolutePath(StringRef File) { 153 StringRef RelativePath(File); 154 // FIXME: Should '.\\' be accepted on Win32? 155 if (RelativePath.startswith("./")) { 156 RelativePath = RelativePath.substr(strlen("./")); 157 } 158 159 SmallString<1024> AbsolutePath = RelativePath; 160 std::error_code EC = llvm::sys::fs::make_absolute(AbsolutePath); 161 assert(!EC); 162 (void)EC; 163 llvm::sys::path::native(AbsolutePath); 164 return AbsolutePath.str(); 165 } 166 167 void addTargetAndModeForProgramName(std::vector<std::string> &CommandLine, 168 StringRef InvokedAs) { 169 if (!CommandLine.empty() && !InvokedAs.empty()) { 170 bool AlreadyHasTarget = false; 171 bool AlreadyHasMode = false; 172 // Skip CommandLine[0]. 173 for (auto Token = ++CommandLine.begin(); Token != CommandLine.end(); 174 ++Token) { 175 StringRef TokenRef(*Token); 176 AlreadyHasTarget |= 177 (TokenRef == "-target" || TokenRef.startswith("-target=")); 178 AlreadyHasMode |= (TokenRef == "--driver-mode" || 179 TokenRef.startswith("--driver-mode=")); 180 } 181 auto TargetMode = 182 clang::driver::ToolChain::getTargetAndModeFromProgramName(InvokedAs); 183 if (!AlreadyHasMode && !TargetMode.second.empty()) { 184 CommandLine.insert(++CommandLine.begin(), TargetMode.second); 185 } 186 if (!AlreadyHasTarget && !TargetMode.first.empty()) { 187 CommandLine.insert(++CommandLine.begin(), {"-target", TargetMode.first}); 188 } 189 } 190 } 191 192 namespace { 193 194 class SingleFrontendActionFactory : public FrontendActionFactory { 195 FrontendAction *Action; 196 197 public: 198 SingleFrontendActionFactory(FrontendAction *Action) : Action(Action) {} 199 200 FrontendAction *create() override { return Action; } 201 }; 202 203 } 204 205 ToolInvocation::ToolInvocation( 206 std::vector<std::string> CommandLine, ToolAction *Action, 207 FileManager *Files, std::shared_ptr<PCHContainerOperations> PCHContainerOps) 208 : CommandLine(std::move(CommandLine)), Action(Action), OwnsAction(false), 209 Files(Files), PCHContainerOps(PCHContainerOps), DiagConsumer(nullptr) {} 210 211 ToolInvocation::ToolInvocation( 212 std::vector<std::string> CommandLine, FrontendAction *FAction, 213 FileManager *Files, std::shared_ptr<PCHContainerOperations> PCHContainerOps) 214 : CommandLine(std::move(CommandLine)), 215 Action(new SingleFrontendActionFactory(FAction)), OwnsAction(true), 216 Files(Files), PCHContainerOps(PCHContainerOps), DiagConsumer(nullptr) {} 217 218 ToolInvocation::~ToolInvocation() { 219 if (OwnsAction) 220 delete Action; 221 } 222 223 void ToolInvocation::mapVirtualFile(StringRef FilePath, StringRef Content) { 224 SmallString<1024> PathStorage; 225 llvm::sys::path::native(FilePath, PathStorage); 226 MappedFileContents[PathStorage] = Content; 227 } 228 229 bool ToolInvocation::run() { 230 std::vector<const char*> Argv; 231 for (const std::string &Str : CommandLine) 232 Argv.push_back(Str.c_str()); 233 const char *const BinaryName = Argv[0]; 234 IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions(); 235 TextDiagnosticPrinter DiagnosticPrinter( 236 llvm::errs(), &*DiagOpts); 237 DiagnosticsEngine Diagnostics( 238 IntrusiveRefCntPtr<clang::DiagnosticIDs>(new DiagnosticIDs()), &*DiagOpts, 239 DiagConsumer ? DiagConsumer : &DiagnosticPrinter, false); 240 241 const std::unique_ptr<clang::driver::Driver> Driver( 242 newDriver(&Diagnostics, BinaryName, Files->getVirtualFileSystem())); 243 // Since the input might only be virtual, don't check whether it exists. 244 Driver->setCheckInputsExist(false); 245 const std::unique_ptr<clang::driver::Compilation> Compilation( 246 Driver->BuildCompilation(llvm::makeArrayRef(Argv))); 247 const llvm::opt::ArgStringList *const CC1Args = getCC1Arguments( 248 &Diagnostics, Compilation.get()); 249 if (!CC1Args) { 250 return false; 251 } 252 std::unique_ptr<clang::CompilerInvocation> Invocation( 253 newInvocation(&Diagnostics, *CC1Args)); 254 // FIXME: remove this when all users have migrated! 255 for (const auto &It : MappedFileContents) { 256 // Inject the code as the given file name into the preprocessor options. 257 std::unique_ptr<llvm::MemoryBuffer> Input = 258 llvm::MemoryBuffer::getMemBuffer(It.getValue()); 259 Invocation->getPreprocessorOpts().addRemappedFile(It.getKey(), 260 Input.release()); 261 } 262 return runInvocation(BinaryName, Compilation.get(), Invocation.release(), 263 PCHContainerOps); 264 } 265 266 bool ToolInvocation::runInvocation( 267 const char *BinaryName, clang::driver::Compilation *Compilation, 268 clang::CompilerInvocation *Invocation, 269 std::shared_ptr<PCHContainerOperations> PCHContainerOps) { 270 // Show the invocation, with -v. 271 if (Invocation->getHeaderSearchOpts().Verbose) { 272 llvm::errs() << "clang Invocation:\n"; 273 Compilation->getJobs().Print(llvm::errs(), "\n", true); 274 llvm::errs() << "\n"; 275 } 276 277 return Action->runInvocation(Invocation, Files, PCHContainerOps, 278 DiagConsumer); 279 } 280 281 bool FrontendActionFactory::runInvocation( 282 CompilerInvocation *Invocation, FileManager *Files, 283 std::shared_ptr<PCHContainerOperations> PCHContainerOps, 284 DiagnosticConsumer *DiagConsumer) { 285 // Create a compiler instance to handle the actual work. 286 clang::CompilerInstance Compiler(PCHContainerOps); 287 Compiler.setInvocation(Invocation); 288 Compiler.setFileManager(Files); 289 290 // The FrontendAction can have lifetime requirements for Compiler or its 291 // members, and we need to ensure it's deleted earlier than Compiler. So we 292 // pass it to an std::unique_ptr declared after the Compiler variable. 293 std::unique_ptr<FrontendAction> ScopedToolAction(create()); 294 295 // Create the compiler's actual diagnostics engine. 296 Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false); 297 if (!Compiler.hasDiagnostics()) 298 return false; 299 300 Compiler.createSourceManager(*Files); 301 302 const bool Success = Compiler.ExecuteAction(*ScopedToolAction); 303 304 Files->clearStatCaches(); 305 return Success; 306 } 307 308 ClangTool::ClangTool(const CompilationDatabase &Compilations, 309 ArrayRef<std::string> SourcePaths, 310 std::shared_ptr<PCHContainerOperations> PCHContainerOps) 311 : Compilations(Compilations), SourcePaths(SourcePaths), 312 PCHContainerOps(PCHContainerOps), 313 OverlayFileSystem(new vfs::OverlayFileSystem(vfs::getRealFileSystem())), 314 InMemoryFileSystem(new vfs::InMemoryFileSystem), 315 Files(new FileManager(FileSystemOptions(), OverlayFileSystem)), 316 DiagConsumer(nullptr) { 317 OverlayFileSystem->pushOverlay(InMemoryFileSystem); 318 appendArgumentsAdjuster(getClangStripOutputAdjuster()); 319 appendArgumentsAdjuster(getClangSyntaxOnlyAdjuster()); 320 } 321 322 ClangTool::~ClangTool() {} 323 324 void ClangTool::mapVirtualFile(StringRef FilePath, StringRef Content) { 325 MappedFileContents.push_back(std::make_pair(FilePath, Content)); 326 } 327 328 void ClangTool::appendArgumentsAdjuster(ArgumentsAdjuster Adjuster) { 329 if (ArgsAdjuster) 330 ArgsAdjuster = combineAdjusters(ArgsAdjuster, Adjuster); 331 else 332 ArgsAdjuster = Adjuster; 333 } 334 335 void ClangTool::clearArgumentsAdjusters() { 336 ArgsAdjuster = nullptr; 337 } 338 339 int ClangTool::run(ToolAction *Action) { 340 // Exists solely for the purpose of lookup of the resource path. 341 // This just needs to be some symbol in the binary. 342 static int StaticSymbol; 343 // The driver detects the builtin header path based on the path of the 344 // executable. 345 // FIXME: On linux, GetMainExecutable is independent of the value of the 346 // first argument, thus allowing ClangTool and runToolOnCode to just 347 // pass in made-up names here. Make sure this works on other platforms. 348 std::string MainExecutable = 349 llvm::sys::fs::getMainExecutable("clang_tool", &StaticSymbol); 350 351 llvm::SmallString<128> InitialDirectory; 352 if (std::error_code EC = llvm::sys::fs::current_path(InitialDirectory)) 353 llvm::report_fatal_error("Cannot detect current path: " + 354 Twine(EC.message())); 355 356 // First insert all absolute paths into the in-memory VFS. These are global 357 // for all compile commands. 358 if (SeenWorkingDirectories.insert("/").second) 359 for (const auto &MappedFile : MappedFileContents) 360 if (llvm::sys::path::is_absolute(MappedFile.first)) 361 InMemoryFileSystem->addFile( 362 MappedFile.first, 0, 363 llvm::MemoryBuffer::getMemBuffer(MappedFile.second)); 364 365 bool ProcessingFailed = false; 366 for (const auto &SourcePath : SourcePaths) { 367 std::string File(getAbsolutePath(SourcePath)); 368 369 // Currently implementations of CompilationDatabase::getCompileCommands can 370 // change the state of the file system (e.g. prepare generated headers), so 371 // this method needs to run right before we invoke the tool, as the next 372 // file may require a different (incompatible) state of the file system. 373 // 374 // FIXME: Make the compilation database interface more explicit about the 375 // requirements to the order of invocation of its members. 376 std::vector<CompileCommand> CompileCommandsForFile = 377 Compilations.getCompileCommands(File); 378 if (CompileCommandsForFile.empty()) { 379 // FIXME: There are two use cases here: doing a fuzzy 380 // "find . -name '*.cc' |xargs tool" match, where as a user I don't care 381 // about the .cc files that were not found, and the use case where I 382 // specify all files I want to run over explicitly, where this should 383 // be an error. We'll want to add an option for this. 384 llvm::errs() << "Skipping " << File << ". Compile command not found.\n"; 385 continue; 386 } 387 for (CompileCommand &CompileCommand : CompileCommandsForFile) { 388 // FIXME: chdir is thread hostile; on the other hand, creating the same 389 // behavior as chdir is complex: chdir resolves the path once, thus 390 // guaranteeing that all subsequent relative path operations work 391 // on the same path the original chdir resulted in. This makes a 392 // difference for example on network filesystems, where symlinks might be 393 // switched during runtime of the tool. Fixing this depends on having a 394 // file system abstraction that allows openat() style interactions. 395 if (OverlayFileSystem->setCurrentWorkingDirectory( 396 CompileCommand.Directory)) 397 llvm::report_fatal_error("Cannot chdir into \"" + 398 Twine(CompileCommand.Directory) + "\n!"); 399 400 // Now fill the in-memory VFS with the relative file mappings so it will 401 // have the correct relative paths. We never remove mappings but that 402 // should be fine. 403 if (SeenWorkingDirectories.insert(CompileCommand.Directory).second) 404 for (const auto &MappedFile : MappedFileContents) 405 if (!llvm::sys::path::is_absolute(MappedFile.first)) 406 InMemoryFileSystem->addFile( 407 MappedFile.first, 0, 408 llvm::MemoryBuffer::getMemBuffer(MappedFile.second)); 409 410 std::vector<std::string> CommandLine = CompileCommand.CommandLine; 411 if (ArgsAdjuster) 412 CommandLine = ArgsAdjuster(CommandLine, CompileCommand.Filename); 413 assert(!CommandLine.empty()); 414 CommandLine[0] = MainExecutable; 415 // FIXME: We need a callback mechanism for the tool writer to output a 416 // customized message for each file. 417 DEBUG({ llvm::dbgs() << "Processing: " << File << ".\n"; }); 418 ToolInvocation Invocation(std::move(CommandLine), Action, Files.get(), 419 PCHContainerOps); 420 Invocation.setDiagnosticConsumer(DiagConsumer); 421 422 if (!Invocation.run()) { 423 // FIXME: Diagnostics should be used instead. 424 llvm::errs() << "Error while processing " << File << ".\n"; 425 ProcessingFailed = true; 426 } 427 // Return to the initial directory to correctly resolve next file by 428 // relative path. 429 if (OverlayFileSystem->setCurrentWorkingDirectory(InitialDirectory.c_str())) 430 llvm::report_fatal_error("Cannot chdir into \"" + 431 Twine(InitialDirectory) + "\n!"); 432 } 433 } 434 return ProcessingFailed ? 1 : 0; 435 } 436 437 namespace { 438 439 class ASTBuilderAction : public ToolAction { 440 std::vector<std::unique_ptr<ASTUnit>> &ASTs; 441 442 public: 443 ASTBuilderAction(std::vector<std::unique_ptr<ASTUnit>> &ASTs) : ASTs(ASTs) {} 444 445 bool runInvocation(CompilerInvocation *Invocation, FileManager *Files, 446 std::shared_ptr<PCHContainerOperations> PCHContainerOps, 447 DiagnosticConsumer *DiagConsumer) override { 448 std::unique_ptr<ASTUnit> AST = ASTUnit::LoadFromCompilerInvocation( 449 Invocation, PCHContainerOps, 450 CompilerInstance::createDiagnostics(&Invocation->getDiagnosticOpts(), 451 DiagConsumer, 452 /*ShouldOwnClient=*/false), 453 Files); 454 if (!AST) 455 return false; 456 457 ASTs.push_back(std::move(AST)); 458 return true; 459 } 460 }; 461 462 } 463 464 int ClangTool::buildASTs(std::vector<std::unique_ptr<ASTUnit>> &ASTs) { 465 ASTBuilderAction Action(ASTs); 466 return run(&Action); 467 } 468 469 std::unique_ptr<ASTUnit> 470 buildASTFromCode(const Twine &Code, const Twine &FileName, 471 std::shared_ptr<PCHContainerOperations> PCHContainerOps) { 472 return buildASTFromCodeWithArgs(Code, std::vector<std::string>(), FileName, 473 PCHContainerOps); 474 } 475 476 std::unique_ptr<ASTUnit> buildASTFromCodeWithArgs( 477 const Twine &Code, const std::vector<std::string> &Args, 478 const Twine &FileName, 479 std::shared_ptr<PCHContainerOperations> PCHContainerOps) { 480 SmallString<16> FileNameStorage; 481 StringRef FileNameRef = FileName.toNullTerminatedStringRef(FileNameStorage); 482 483 std::vector<std::unique_ptr<ASTUnit>> ASTs; 484 ASTBuilderAction Action(ASTs); 485 llvm::IntrusiveRefCntPtr<vfs::OverlayFileSystem> OverlayFileSystem( 486 new vfs::OverlayFileSystem(vfs::getRealFileSystem())); 487 llvm::IntrusiveRefCntPtr<vfs::InMemoryFileSystem> InMemoryFileSystem( 488 new vfs::InMemoryFileSystem); 489 OverlayFileSystem->pushOverlay(InMemoryFileSystem); 490 llvm::IntrusiveRefCntPtr<FileManager> Files( 491 new FileManager(FileSystemOptions(), OverlayFileSystem)); 492 ToolInvocation Invocation(getSyntaxOnlyToolArgs(Args, FileNameRef), &Action, 493 Files.get(), PCHContainerOps); 494 495 SmallString<1024> CodeStorage; 496 InMemoryFileSystem->addFile(FileNameRef, 0, 497 llvm::MemoryBuffer::getMemBuffer( 498 Code.toNullTerminatedStringRef(CodeStorage))); 499 if (!Invocation.run()) 500 return nullptr; 501 502 assert(ASTs.size() == 1); 503 return std::move(ASTs[0]); 504 } 505 506 } // end namespace tooling 507 } // end namespace clang 508