1 //===--- Tooling.cpp - Running clang standalone tools ---------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file implements functions to run clang tools standalone instead 11 // of running them as a plugin. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "clang/Tooling/Tooling.h" 16 #include "clang/AST/ASTConsumer.h" 17 #include "clang/Driver/Compilation.h" 18 #include "clang/Driver/Driver.h" 19 #include "clang/Driver/Tool.h" 20 #include "clang/Driver/ToolChain.h" 21 #include "clang/Frontend/ASTUnit.h" 22 #include "clang/Frontend/CompilerInstance.h" 23 #include "clang/Frontend/FrontendDiagnostic.h" 24 #include "clang/Frontend/TextDiagnosticPrinter.h" 25 #include "clang/Tooling/ArgumentsAdjusters.h" 26 #include "clang/Tooling/CompilationDatabase.h" 27 #include "llvm/ADT/STLExtras.h" 28 #include "llvm/Config/llvm-config.h" 29 #include "llvm/Option/Option.h" 30 #include "llvm/Support/Debug.h" 31 #include "llvm/Support/FileSystem.h" 32 #include "llvm/Support/Host.h" 33 #include "llvm/Support/raw_ostream.h" 34 #include <utility> 35 36 #define DEBUG_TYPE "clang-tooling" 37 38 namespace clang { 39 namespace tooling { 40 41 ToolAction::~ToolAction() {} 42 43 FrontendActionFactory::~FrontendActionFactory() {} 44 45 // FIXME: This file contains structural duplication with other parts of the 46 // code that sets up a compiler to run tools on it, and we should refactor 47 // it to be based on the same framework. 48 49 /// \brief Builds a clang driver initialized for running clang tools. 50 static clang::driver::Driver *newDriver( 51 clang::DiagnosticsEngine *Diagnostics, const char *BinaryName, 52 IntrusiveRefCntPtr<vfs::FileSystem> VFS) { 53 clang::driver::Driver *CompilerDriver = 54 new clang::driver::Driver(BinaryName, llvm::sys::getDefaultTargetTriple(), 55 *Diagnostics, std::move(VFS)); 56 CompilerDriver->setTitle("clang_based_tool"); 57 return CompilerDriver; 58 } 59 60 /// \brief Retrieves the clang CC1 specific flags out of the compilation's jobs. 61 /// 62 /// Returns NULL on error. 63 static const llvm::opt::ArgStringList *getCC1Arguments( 64 clang::DiagnosticsEngine *Diagnostics, 65 clang::driver::Compilation *Compilation) { 66 // We expect to get back exactly one Command job, if we didn't something 67 // failed. Extract that job from the Compilation. 68 const clang::driver::JobList &Jobs = Compilation->getJobs(); 69 if (Jobs.size() != 1 || !isa<clang::driver::Command>(*Jobs.begin())) { 70 SmallString<256> error_msg; 71 llvm::raw_svector_ostream error_stream(error_msg); 72 Jobs.Print(error_stream, "; ", true); 73 Diagnostics->Report(clang::diag::err_fe_expected_compiler_job) 74 << error_stream.str(); 75 return nullptr; 76 } 77 78 // The one job we find should be to invoke clang again. 79 const clang::driver::Command &Cmd = 80 cast<clang::driver::Command>(*Jobs.begin()); 81 if (StringRef(Cmd.getCreator().getName()) != "clang") { 82 Diagnostics->Report(clang::diag::err_fe_expected_clang_command); 83 return nullptr; 84 } 85 86 return &Cmd.getArguments(); 87 } 88 89 /// \brief Returns a clang build invocation initialized from the CC1 flags. 90 clang::CompilerInvocation *newInvocation( 91 clang::DiagnosticsEngine *Diagnostics, 92 const llvm::opt::ArgStringList &CC1Args) { 93 assert(!CC1Args.empty() && "Must at least contain the program name!"); 94 clang::CompilerInvocation *Invocation = new clang::CompilerInvocation; 95 clang::CompilerInvocation::CreateFromArgs( 96 *Invocation, CC1Args.data() + 1, CC1Args.data() + CC1Args.size(), 97 *Diagnostics); 98 Invocation->getFrontendOpts().DisableFree = false; 99 Invocation->getCodeGenOpts().DisableFree = false; 100 Invocation->getDependencyOutputOpts() = DependencyOutputOptions(); 101 return Invocation; 102 } 103 104 bool runToolOnCode(clang::FrontendAction *ToolAction, const Twine &Code, 105 const Twine &FileName, 106 std::shared_ptr<PCHContainerOperations> PCHContainerOps) { 107 return runToolOnCodeWithArgs(ToolAction, Code, std::vector<std::string>(), 108 FileName, "clang-tool", 109 std::move(PCHContainerOps)); 110 } 111 112 static std::vector<std::string> 113 getSyntaxOnlyToolArgs(const Twine &ToolName, 114 const std::vector<std::string> &ExtraArgs, 115 StringRef FileName) { 116 std::vector<std::string> Args; 117 Args.push_back(ToolName.str()); 118 Args.push_back("-fsyntax-only"); 119 Args.insert(Args.end(), ExtraArgs.begin(), ExtraArgs.end()); 120 Args.push_back(FileName.str()); 121 return Args; 122 } 123 124 bool runToolOnCodeWithArgs( 125 clang::FrontendAction *ToolAction, const Twine &Code, 126 const std::vector<std::string> &Args, const Twine &FileName, 127 const Twine &ToolName, 128 std::shared_ptr<PCHContainerOperations> PCHContainerOps, 129 const FileContentMappings &VirtualMappedFiles) { 130 131 SmallString<16> FileNameStorage; 132 StringRef FileNameRef = FileName.toNullTerminatedStringRef(FileNameStorage); 133 llvm::IntrusiveRefCntPtr<vfs::OverlayFileSystem> OverlayFileSystem( 134 new vfs::OverlayFileSystem(vfs::getRealFileSystem())); 135 llvm::IntrusiveRefCntPtr<vfs::InMemoryFileSystem> InMemoryFileSystem( 136 new vfs::InMemoryFileSystem); 137 OverlayFileSystem->pushOverlay(InMemoryFileSystem); 138 llvm::IntrusiveRefCntPtr<FileManager> Files( 139 new FileManager(FileSystemOptions(), OverlayFileSystem)); 140 ToolInvocation Invocation(getSyntaxOnlyToolArgs(ToolName, Args, FileNameRef), 141 ToolAction, Files.get(), 142 std::move(PCHContainerOps)); 143 144 SmallString<1024> CodeStorage; 145 InMemoryFileSystem->addFile(FileNameRef, 0, 146 llvm::MemoryBuffer::getMemBuffer( 147 Code.toNullTerminatedStringRef(CodeStorage))); 148 149 for (auto &FilenameWithContent : VirtualMappedFiles) { 150 InMemoryFileSystem->addFile( 151 FilenameWithContent.first, 0, 152 llvm::MemoryBuffer::getMemBuffer(FilenameWithContent.second)); 153 } 154 155 return Invocation.run(); 156 } 157 158 std::string getAbsolutePath(StringRef File) { 159 StringRef RelativePath(File); 160 // FIXME: Should '.\\' be accepted on Win32? 161 if (RelativePath.startswith("./")) { 162 RelativePath = RelativePath.substr(strlen("./")); 163 } 164 165 SmallString<1024> AbsolutePath = RelativePath; 166 std::error_code EC = llvm::sys::fs::make_absolute(AbsolutePath); 167 assert(!EC); 168 (void)EC; 169 llvm::sys::path::native(AbsolutePath); 170 return AbsolutePath.str(); 171 } 172 173 void addTargetAndModeForProgramName(std::vector<std::string> &CommandLine, 174 StringRef InvokedAs) { 175 if (!CommandLine.empty() && !InvokedAs.empty()) { 176 bool AlreadyHasTarget = false; 177 bool AlreadyHasMode = false; 178 // Skip CommandLine[0]. 179 for (auto Token = ++CommandLine.begin(); Token != CommandLine.end(); 180 ++Token) { 181 StringRef TokenRef(*Token); 182 AlreadyHasTarget |= 183 (TokenRef == "-target" || TokenRef.startswith("-target=")); 184 AlreadyHasMode |= (TokenRef == "--driver-mode" || 185 TokenRef.startswith("--driver-mode=")); 186 } 187 auto TargetMode = 188 clang::driver::ToolChain::getTargetAndModeFromProgramName(InvokedAs); 189 if (!AlreadyHasMode && !TargetMode.second.empty()) { 190 CommandLine.insert(++CommandLine.begin(), TargetMode.second); 191 } 192 if (!AlreadyHasTarget && !TargetMode.first.empty()) { 193 CommandLine.insert(++CommandLine.begin(), {"-target", TargetMode.first}); 194 } 195 } 196 } 197 198 namespace { 199 200 class SingleFrontendActionFactory : public FrontendActionFactory { 201 FrontendAction *Action; 202 203 public: 204 SingleFrontendActionFactory(FrontendAction *Action) : Action(Action) {} 205 206 FrontendAction *create() override { return Action; } 207 }; 208 209 } 210 211 ToolInvocation::ToolInvocation( 212 std::vector<std::string> CommandLine, ToolAction *Action, 213 FileManager *Files, std::shared_ptr<PCHContainerOperations> PCHContainerOps) 214 : CommandLine(std::move(CommandLine)), Action(Action), OwnsAction(false), 215 Files(Files), PCHContainerOps(std::move(PCHContainerOps)), 216 DiagConsumer(nullptr) {} 217 218 ToolInvocation::ToolInvocation( 219 std::vector<std::string> CommandLine, FrontendAction *FAction, 220 FileManager *Files, std::shared_ptr<PCHContainerOperations> PCHContainerOps) 221 : CommandLine(std::move(CommandLine)), 222 Action(new SingleFrontendActionFactory(FAction)), OwnsAction(true), 223 Files(Files), PCHContainerOps(std::move(PCHContainerOps)), 224 DiagConsumer(nullptr) {} 225 226 ToolInvocation::~ToolInvocation() { 227 if (OwnsAction) 228 delete Action; 229 } 230 231 void ToolInvocation::mapVirtualFile(StringRef FilePath, StringRef Content) { 232 SmallString<1024> PathStorage; 233 llvm::sys::path::native(FilePath, PathStorage); 234 MappedFileContents[PathStorage] = Content; 235 } 236 237 bool ToolInvocation::run() { 238 std::vector<const char*> Argv; 239 for (const std::string &Str : CommandLine) 240 Argv.push_back(Str.c_str()); 241 const char *const BinaryName = Argv[0]; 242 IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions(); 243 TextDiagnosticPrinter DiagnosticPrinter( 244 llvm::errs(), &*DiagOpts); 245 DiagnosticsEngine Diagnostics( 246 IntrusiveRefCntPtr<clang::DiagnosticIDs>(new DiagnosticIDs()), &*DiagOpts, 247 DiagConsumer ? DiagConsumer : &DiagnosticPrinter, false); 248 249 const std::unique_ptr<clang::driver::Driver> Driver( 250 newDriver(&Diagnostics, BinaryName, Files->getVirtualFileSystem())); 251 // Since the input might only be virtual, don't check whether it exists. 252 Driver->setCheckInputsExist(false); 253 const std::unique_ptr<clang::driver::Compilation> Compilation( 254 Driver->BuildCompilation(llvm::makeArrayRef(Argv))); 255 const llvm::opt::ArgStringList *const CC1Args = getCC1Arguments( 256 &Diagnostics, Compilation.get()); 257 if (!CC1Args) { 258 return false; 259 } 260 std::unique_ptr<clang::CompilerInvocation> Invocation( 261 newInvocation(&Diagnostics, *CC1Args)); 262 // FIXME: remove this when all users have migrated! 263 for (const auto &It : MappedFileContents) { 264 // Inject the code as the given file name into the preprocessor options. 265 std::unique_ptr<llvm::MemoryBuffer> Input = 266 llvm::MemoryBuffer::getMemBuffer(It.getValue()); 267 Invocation->getPreprocessorOpts().addRemappedFile(It.getKey(), 268 Input.release()); 269 } 270 return runInvocation(BinaryName, Compilation.get(), Invocation.release(), 271 std::move(PCHContainerOps)); 272 } 273 274 bool ToolInvocation::runInvocation( 275 const char *BinaryName, clang::driver::Compilation *Compilation, 276 clang::CompilerInvocation *Invocation, 277 std::shared_ptr<PCHContainerOperations> PCHContainerOps) { 278 // Show the invocation, with -v. 279 if (Invocation->getHeaderSearchOpts().Verbose) { 280 llvm::errs() << "clang Invocation:\n"; 281 Compilation->getJobs().Print(llvm::errs(), "\n", true); 282 llvm::errs() << "\n"; 283 } 284 285 return Action->runInvocation(Invocation, Files, std::move(PCHContainerOps), 286 DiagConsumer); 287 } 288 289 bool FrontendActionFactory::runInvocation( 290 CompilerInvocation *Invocation, FileManager *Files, 291 std::shared_ptr<PCHContainerOperations> PCHContainerOps, 292 DiagnosticConsumer *DiagConsumer) { 293 // Create a compiler instance to handle the actual work. 294 clang::CompilerInstance Compiler(std::move(PCHContainerOps)); 295 Compiler.setInvocation(Invocation); 296 Compiler.setFileManager(Files); 297 298 // The FrontendAction can have lifetime requirements for Compiler or its 299 // members, and we need to ensure it's deleted earlier than Compiler. So we 300 // pass it to an std::unique_ptr declared after the Compiler variable. 301 std::unique_ptr<FrontendAction> ScopedToolAction(create()); 302 303 // Create the compiler's actual diagnostics engine. 304 Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false); 305 if (!Compiler.hasDiagnostics()) 306 return false; 307 308 Compiler.createSourceManager(*Files); 309 310 const bool Success = Compiler.ExecuteAction(*ScopedToolAction); 311 312 Files->clearStatCaches(); 313 return Success; 314 } 315 316 ClangTool::ClangTool(const CompilationDatabase &Compilations, 317 ArrayRef<std::string> SourcePaths, 318 std::shared_ptr<PCHContainerOperations> PCHContainerOps) 319 : Compilations(Compilations), SourcePaths(SourcePaths), 320 PCHContainerOps(std::move(PCHContainerOps)), 321 OverlayFileSystem(new vfs::OverlayFileSystem(vfs::getRealFileSystem())), 322 InMemoryFileSystem(new vfs::InMemoryFileSystem), 323 Files(new FileManager(FileSystemOptions(), OverlayFileSystem)), 324 DiagConsumer(nullptr) { 325 OverlayFileSystem->pushOverlay(InMemoryFileSystem); 326 appendArgumentsAdjuster(getClangStripOutputAdjuster()); 327 appendArgumentsAdjuster(getClangSyntaxOnlyAdjuster()); 328 } 329 330 ClangTool::~ClangTool() {} 331 332 void ClangTool::mapVirtualFile(StringRef FilePath, StringRef Content) { 333 MappedFileContents.push_back(std::make_pair(FilePath, Content)); 334 } 335 336 void ClangTool::appendArgumentsAdjuster(ArgumentsAdjuster Adjuster) { 337 if (ArgsAdjuster) 338 ArgsAdjuster = 339 combineAdjusters(std::move(ArgsAdjuster), std::move(Adjuster)); 340 else 341 ArgsAdjuster = std::move(Adjuster); 342 } 343 344 void ClangTool::clearArgumentsAdjusters() { 345 ArgsAdjuster = nullptr; 346 } 347 348 static void injectResourceDir(CommandLineArguments &Args, const char *Argv0, 349 void *MainAddr) { 350 // Allow users to override the resource dir. 351 for (StringRef Arg : Args) 352 if (Arg.startswith("-resource-dir")) 353 return; 354 355 // If there's no override in place add our resource dir. 356 Args.push_back("-resource-dir=" + 357 CompilerInvocation::GetResourcesPath(Argv0, MainAddr)); 358 } 359 360 int ClangTool::run(ToolAction *Action) { 361 // Exists solely for the purpose of lookup of the resource path. 362 // This just needs to be some symbol in the binary. 363 static int StaticSymbol; 364 365 llvm::SmallString<128> InitialDirectory; 366 if (std::error_code EC = llvm::sys::fs::current_path(InitialDirectory)) 367 llvm::report_fatal_error("Cannot detect current path: " + 368 Twine(EC.message())); 369 370 // First insert all absolute paths into the in-memory VFS. These are global 371 // for all compile commands. 372 if (SeenWorkingDirectories.insert("/").second) 373 for (const auto &MappedFile : MappedFileContents) 374 if (llvm::sys::path::is_absolute(MappedFile.first)) 375 InMemoryFileSystem->addFile( 376 MappedFile.first, 0, 377 llvm::MemoryBuffer::getMemBuffer(MappedFile.second)); 378 379 bool ProcessingFailed = false; 380 for (const auto &SourcePath : SourcePaths) { 381 std::string File(getAbsolutePath(SourcePath)); 382 383 // Currently implementations of CompilationDatabase::getCompileCommands can 384 // change the state of the file system (e.g. prepare generated headers), so 385 // this method needs to run right before we invoke the tool, as the next 386 // file may require a different (incompatible) state of the file system. 387 // 388 // FIXME: Make the compilation database interface more explicit about the 389 // requirements to the order of invocation of its members. 390 std::vector<CompileCommand> CompileCommandsForFile = 391 Compilations.getCompileCommands(File); 392 if (CompileCommandsForFile.empty()) { 393 // FIXME: There are two use cases here: doing a fuzzy 394 // "find . -name '*.cc' |xargs tool" match, where as a user I don't care 395 // about the .cc files that were not found, and the use case where I 396 // specify all files I want to run over explicitly, where this should 397 // be an error. We'll want to add an option for this. 398 llvm::errs() << "Skipping " << File << ". Compile command not found.\n"; 399 continue; 400 } 401 for (CompileCommand &CompileCommand : CompileCommandsForFile) { 402 // FIXME: chdir is thread hostile; on the other hand, creating the same 403 // behavior as chdir is complex: chdir resolves the path once, thus 404 // guaranteeing that all subsequent relative path operations work 405 // on the same path the original chdir resulted in. This makes a 406 // difference for example on network filesystems, where symlinks might be 407 // switched during runtime of the tool. Fixing this depends on having a 408 // file system abstraction that allows openat() style interactions. 409 if (OverlayFileSystem->setCurrentWorkingDirectory( 410 CompileCommand.Directory)) 411 llvm::report_fatal_error("Cannot chdir into \"" + 412 Twine(CompileCommand.Directory) + "\n!"); 413 414 // Now fill the in-memory VFS with the relative file mappings so it will 415 // have the correct relative paths. We never remove mappings but that 416 // should be fine. 417 if (SeenWorkingDirectories.insert(CompileCommand.Directory).second) 418 for (const auto &MappedFile : MappedFileContents) 419 if (!llvm::sys::path::is_absolute(MappedFile.first)) 420 InMemoryFileSystem->addFile( 421 MappedFile.first, 0, 422 llvm::MemoryBuffer::getMemBuffer(MappedFile.second)); 423 424 std::vector<std::string> CommandLine = CompileCommand.CommandLine; 425 if (ArgsAdjuster) 426 CommandLine = ArgsAdjuster(CommandLine, CompileCommand.Filename); 427 assert(!CommandLine.empty()); 428 429 // Add the resource dir based on the binary of this tool. argv[0] in the 430 // compilation database may refer to a different compiler and we want to 431 // pick up the very same standard library that compiler is using. The 432 // builtin headers in the resource dir need to match the exact clang 433 // version the tool is using. 434 // FIXME: On linux, GetMainExecutable is independent of the value of the 435 // first argument, thus allowing ClangTool and runToolOnCode to just 436 // pass in made-up names here. Make sure this works on other platforms. 437 injectResourceDir(CommandLine, "clang_tool", &StaticSymbol); 438 439 // FIXME: We need a callback mechanism for the tool writer to output a 440 // customized message for each file. 441 DEBUG({ llvm::dbgs() << "Processing: " << File << ".\n"; }); 442 ToolInvocation Invocation(std::move(CommandLine), Action, Files.get(), 443 PCHContainerOps); 444 Invocation.setDiagnosticConsumer(DiagConsumer); 445 446 if (!Invocation.run()) { 447 // FIXME: Diagnostics should be used instead. 448 llvm::errs() << "Error while processing " << File << ".\n"; 449 ProcessingFailed = true; 450 } 451 // Return to the initial directory to correctly resolve next file by 452 // relative path. 453 if (OverlayFileSystem->setCurrentWorkingDirectory(InitialDirectory.c_str())) 454 llvm::report_fatal_error("Cannot chdir into \"" + 455 Twine(InitialDirectory) + "\n!"); 456 } 457 } 458 return ProcessingFailed ? 1 : 0; 459 } 460 461 namespace { 462 463 class ASTBuilderAction : public ToolAction { 464 std::vector<std::unique_ptr<ASTUnit>> &ASTs; 465 466 public: 467 ASTBuilderAction(std::vector<std::unique_ptr<ASTUnit>> &ASTs) : ASTs(ASTs) {} 468 469 bool runInvocation(CompilerInvocation *Invocation, FileManager *Files, 470 std::shared_ptr<PCHContainerOperations> PCHContainerOps, 471 DiagnosticConsumer *DiagConsumer) override { 472 std::unique_ptr<ASTUnit> AST = ASTUnit::LoadFromCompilerInvocation( 473 Invocation, std::move(PCHContainerOps), 474 CompilerInstance::createDiagnostics(&Invocation->getDiagnosticOpts(), 475 DiagConsumer, 476 /*ShouldOwnClient=*/false), 477 Files); 478 if (!AST) 479 return false; 480 481 ASTs.push_back(std::move(AST)); 482 return true; 483 } 484 }; 485 } 486 487 int ClangTool::buildASTs(std::vector<std::unique_ptr<ASTUnit>> &ASTs) { 488 ASTBuilderAction Action(ASTs); 489 return run(&Action); 490 } 491 492 std::unique_ptr<ASTUnit> 493 buildASTFromCode(const Twine &Code, const Twine &FileName, 494 std::shared_ptr<PCHContainerOperations> PCHContainerOps) { 495 return buildASTFromCodeWithArgs(Code, std::vector<std::string>(), FileName, 496 "clang-tool", std::move(PCHContainerOps)); 497 } 498 499 std::unique_ptr<ASTUnit> buildASTFromCodeWithArgs( 500 const Twine &Code, const std::vector<std::string> &Args, 501 const Twine &FileName, const Twine &ToolName, 502 std::shared_ptr<PCHContainerOperations> PCHContainerOps) { 503 SmallString<16> FileNameStorage; 504 StringRef FileNameRef = FileName.toNullTerminatedStringRef(FileNameStorage); 505 506 std::vector<std::unique_ptr<ASTUnit>> ASTs; 507 ASTBuilderAction Action(ASTs); 508 llvm::IntrusiveRefCntPtr<vfs::OverlayFileSystem> OverlayFileSystem( 509 new vfs::OverlayFileSystem(vfs::getRealFileSystem())); 510 llvm::IntrusiveRefCntPtr<vfs::InMemoryFileSystem> InMemoryFileSystem( 511 new vfs::InMemoryFileSystem); 512 OverlayFileSystem->pushOverlay(InMemoryFileSystem); 513 llvm::IntrusiveRefCntPtr<FileManager> Files( 514 new FileManager(FileSystemOptions(), OverlayFileSystem)); 515 ToolInvocation Invocation(getSyntaxOnlyToolArgs(ToolName, Args, FileNameRef), 516 &Action, Files.get(), std::move(PCHContainerOps)); 517 518 SmallString<1024> CodeStorage; 519 InMemoryFileSystem->addFile(FileNameRef, 0, 520 llvm::MemoryBuffer::getMemBuffer( 521 Code.toNullTerminatedStringRef(CodeStorage))); 522 if (!Invocation.run()) 523 return nullptr; 524 525 assert(ASTs.size() == 1); 526 return std::move(ASTs[0]); 527 } 528 529 } // end namespace tooling 530 } // end namespace clang 531