Home | History | Annotate | Download | only in Tooling 
      1 //===--- Tooling.cpp - Running clang standalone tools ---------------------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 //  This file implements functions to run clang tools standalone instead
     11 //  of running them as a plugin.
     12 //
     13 //===----------------------------------------------------------------------===//
     14 
     15 #include "clang/Tooling/Tooling.h"
     16 #include "clang/AST/ASTConsumer.h"
     17 #include "clang/Driver/Compilation.h"
     18 #include "clang/Driver/Driver.h"
     19 #include "clang/Driver/Tool.h"
     20 #include "clang/Frontend/ASTUnit.h"
     21 #include "clang/Frontend/CompilerInstance.h"
     22 #include "clang/Frontend/FrontendDiagnostic.h"
     23 #include "clang/Frontend/TextDiagnosticPrinter.h"
     24 #include "clang/Tooling/ArgumentsAdjusters.h"
     25 #include "clang/Tooling/CompilationDatabase.h"
     26 #include "llvm/ADT/STLExtras.h"
     27 #include "llvm/Config/llvm-config.h"
     28 #include "llvm/Option/Option.h"
     29 #include "llvm/Support/Debug.h"
     30 #include "llvm/Support/FileSystem.h"
     31 #include "llvm/Support/Host.h"
     32 #include "llvm/Support/raw_ostream.h"
     33 
     34 // For chdir, see the comment in ClangTool::run for more information.
     35 #ifdef LLVM_ON_WIN32
     36 #  include <direct.h>
     37 #else
     38 #  include <unistd.h>
     39 #endif
     40 
     41 #define DEBUG_TYPE "clang-tooling"
     42 
     43 namespace clang {
     44 namespace tooling {
     45 
     46 ToolAction::~ToolAction() {}
     47 
     48 FrontendActionFactory::~FrontendActionFactory() {}
     49 
     50 // FIXME: This file contains structural duplication with other parts of the
     51 // code that sets up a compiler to run tools on it, and we should refactor
     52 // it to be based on the same framework.
     53 
     54 /// \brief Builds a clang driver initialized for running clang tools.
     55 static clang::driver::Driver *newDriver(clang::DiagnosticsEngine *Diagnostics,
     56                                         const char *BinaryName) {
     57   clang::driver::Driver *CompilerDriver = new clang::driver::Driver(
     58       BinaryName, llvm::sys::getDefaultTargetTriple(), *Diagnostics);
     59   CompilerDriver->setTitle("clang_based_tool");
     60   return CompilerDriver;
     61 }
     62 
     63 /// \brief Retrieves the clang CC1 specific flags out of the compilation's jobs.
     64 ///
     65 /// Returns NULL on error.
     66 static const llvm::opt::ArgStringList *getCC1Arguments(
     67     clang::DiagnosticsEngine *Diagnostics,
     68     clang::driver::Compilation *Compilation) {
     69   // We expect to get back exactly one Command job, if we didn't something
     70   // failed. Extract that job from the Compilation.
     71   const clang::driver::JobList &Jobs = Compilation->getJobs();
     72   if (Jobs.size() != 1 || !isa<clang::driver::Command>(*Jobs.begin())) {
     73     SmallString<256> error_msg;
     74     llvm::raw_svector_ostream error_stream(error_msg);
     75     Jobs.Print(error_stream, "; ", true);
     76     Diagnostics->Report(clang::diag::err_fe_expected_compiler_job)
     77         << error_stream.str();
     78     return nullptr;
     79   }
     80 
     81   // The one job we find should be to invoke clang again.
     82   const clang::driver::Command &Cmd =
     83       cast<clang::driver::Command>(*Jobs.begin());
     84   if (StringRef(Cmd.getCreator().getName()) != "clang") {
     85     Diagnostics->Report(clang::diag::err_fe_expected_clang_command);
     86     return nullptr;
     87   }
     88 
     89   return &Cmd.getArguments();
     90 }
     91 
     92 /// \brief Returns a clang build invocation initialized from the CC1 flags.
     93 clang::CompilerInvocation *newInvocation(
     94     clang::DiagnosticsEngine *Diagnostics,
     95     const llvm::opt::ArgStringList &CC1Args) {
     96   assert(!CC1Args.empty() && "Must at least contain the program name!");
     97   clang::CompilerInvocation *Invocation = new clang::CompilerInvocation;
     98   clang::CompilerInvocation::CreateFromArgs(
     99       *Invocation, CC1Args.data() + 1, CC1Args.data() + CC1Args.size(),
    100       *Diagnostics);
    101   Invocation->getFrontendOpts().DisableFree = false;
    102   Invocation->getCodeGenOpts().DisableFree = false;
    103   Invocation->getDependencyOutputOpts() = DependencyOutputOptions();
    104   return Invocation;
    105 }
    106 
    107 bool runToolOnCode(clang::FrontendAction *ToolAction, const Twine &Code,
    108                    const Twine &FileName) {
    109   return runToolOnCodeWithArgs(
    110       ToolAction, Code, std::vector<std::string>(), FileName);
    111 }
    112 
    113 static std::vector<std::string>
    114 getSyntaxOnlyToolArgs(const std::vector<std::string> &ExtraArgs,
    115                       StringRef FileName) {
    116   std::vector<std::string> Args;
    117   Args.push_back("clang-tool");
    118   Args.push_back("-fsyntax-only");
    119   Args.insert(Args.end(), ExtraArgs.begin(), ExtraArgs.end());
    120   Args.push_back(FileName.str());
    121   return Args;
    122 }
    123 
    124 bool runToolOnCodeWithArgs(clang::FrontendAction *ToolAction, const Twine &Code,
    125                            const std::vector<std::string> &Args,
    126                            const Twine &FileName,
    127                            const FileContentMappings &VirtualMappedFiles) {
    128 
    129   SmallString<16> FileNameStorage;
    130   StringRef FileNameRef = FileName.toNullTerminatedStringRef(FileNameStorage);
    131   llvm::IntrusiveRefCntPtr<FileManager> Files(
    132       new FileManager(FileSystemOptions()));
    133   ToolInvocation Invocation(getSyntaxOnlyToolArgs(Args, FileNameRef),
    134                             ToolAction, Files.get());
    135 
    136   SmallString<1024> CodeStorage;
    137   Invocation.mapVirtualFile(FileNameRef,
    138                             Code.toNullTerminatedStringRef(CodeStorage));
    139 
    140   for (auto &FilenameWithContent : VirtualMappedFiles) {
    141     Invocation.mapVirtualFile(FilenameWithContent.first,
    142                               FilenameWithContent.second);
    143   }
    144 
    145   return Invocation.run();
    146 }
    147 
    148 std::string getAbsolutePath(StringRef File) {
    149   StringRef RelativePath(File);
    150   // FIXME: Should '.\\' be accepted on Win32?
    151   if (RelativePath.startswith("./")) {
    152     RelativePath = RelativePath.substr(strlen("./"));
    153   }
    154 
    155   SmallString<1024> AbsolutePath = RelativePath;
    156   std::error_code EC = llvm::sys::fs::make_absolute(AbsolutePath);
    157   assert(!EC);
    158   (void)EC;
    159   llvm::sys::path::native(AbsolutePath);
    160   return AbsolutePath.str();
    161 }
    162 
    163 namespace {
    164 
    165 class SingleFrontendActionFactory : public FrontendActionFactory {
    166   FrontendAction *Action;
    167 
    168 public:
    169   SingleFrontendActionFactory(FrontendAction *Action) : Action(Action) {}
    170 
    171   FrontendAction *create() override { return Action; }
    172 };
    173 
    174 }
    175 
    176 ToolInvocation::ToolInvocation(std::vector<std::string> CommandLine,
    177                                ToolAction *Action, FileManager *Files)
    178     : CommandLine(std::move(CommandLine)),
    179       Action(Action),
    180       OwnsAction(false),
    181       Files(Files),
    182       DiagConsumer(nullptr) {}
    183 
    184 ToolInvocation::ToolInvocation(std::vector<std::string> CommandLine,
    185                                FrontendAction *FAction, FileManager *Files)
    186     : CommandLine(std::move(CommandLine)),
    187       Action(new SingleFrontendActionFactory(FAction)),
    188       OwnsAction(true),
    189       Files(Files),
    190       DiagConsumer(nullptr) {}
    191 
    192 ToolInvocation::~ToolInvocation() {
    193   if (OwnsAction)
    194     delete Action;
    195 }
    196 
    197 void ToolInvocation::mapVirtualFile(StringRef FilePath, StringRef Content) {
    198   SmallString<1024> PathStorage;
    199   llvm::sys::path::native(FilePath, PathStorage);
    200   MappedFileContents[PathStorage] = Content;
    201 }
    202 
    203 bool ToolInvocation::run() {
    204   std::vector<const char*> Argv;
    205   for (const std::string &Str : CommandLine)
    206     Argv.push_back(Str.c_str());
    207   const char *const BinaryName = Argv[0];
    208   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
    209   TextDiagnosticPrinter DiagnosticPrinter(
    210       llvm::errs(), &*DiagOpts);
    211   DiagnosticsEngine Diagnostics(
    212       IntrusiveRefCntPtr<clang::DiagnosticIDs>(new DiagnosticIDs()), &*DiagOpts,
    213       DiagConsumer ? DiagConsumer : &DiagnosticPrinter, false);
    214 
    215   const std::unique_ptr<clang::driver::Driver> Driver(
    216       newDriver(&Diagnostics, BinaryName));
    217   // Since the input might only be virtual, don't check whether it exists.
    218   Driver->setCheckInputsExist(false);
    219   const std::unique_ptr<clang::driver::Compilation> Compilation(
    220       Driver->BuildCompilation(llvm::makeArrayRef(Argv)));
    221   const llvm::opt::ArgStringList *const CC1Args = getCC1Arguments(
    222       &Diagnostics, Compilation.get());
    223   if (!CC1Args) {
    224     return false;
    225   }
    226   std::unique_ptr<clang::CompilerInvocation> Invocation(
    227       newInvocation(&Diagnostics, *CC1Args));
    228   for (const auto &It : MappedFileContents) {
    229     // Inject the code as the given file name into the preprocessor options.
    230     std::unique_ptr<llvm::MemoryBuffer> Input =
    231         llvm::MemoryBuffer::getMemBuffer(It.getValue());
    232     Invocation->getPreprocessorOpts().addRemappedFile(It.getKey(),
    233                                                       Input.release());
    234   }
    235   return runInvocation(BinaryName, Compilation.get(), Invocation.release());
    236 }
    237 
    238 bool ToolInvocation::runInvocation(
    239     const char *BinaryName,
    240     clang::driver::Compilation *Compilation,
    241     clang::CompilerInvocation *Invocation) {
    242   // Show the invocation, with -v.
    243   if (Invocation->getHeaderSearchOpts().Verbose) {
    244     llvm::errs() << "clang Invocation:\n";
    245     Compilation->getJobs().Print(llvm::errs(), "\n", true);
    246     llvm::errs() << "\n";
    247   }
    248 
    249   return Action->runInvocation(Invocation, Files, DiagConsumer);
    250 }
    251 
    252 bool FrontendActionFactory::runInvocation(CompilerInvocation *Invocation,
    253                                           FileManager *Files,
    254                                           DiagnosticConsumer *DiagConsumer) {
    255   // Create a compiler instance to handle the actual work.
    256   clang::CompilerInstance Compiler;
    257   Compiler.setInvocation(Invocation);
    258   Compiler.setFileManager(Files);
    259 
    260   // The FrontendAction can have lifetime requirements for Compiler or its
    261   // members, and we need to ensure it's deleted earlier than Compiler. So we
    262   // pass it to an std::unique_ptr declared after the Compiler variable.
    263   std::unique_ptr<FrontendAction> ScopedToolAction(create());
    264 
    265   // Create the compiler's actual diagnostics engine.
    266   Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
    267   if (!Compiler.hasDiagnostics())
    268     return false;
    269 
    270   Compiler.createSourceManager(*Files);
    271 
    272   const bool Success = Compiler.ExecuteAction(*ScopedToolAction);
    273 
    274   Files->clearStatCaches();
    275   return Success;
    276 }
    277 
    278 ClangTool::ClangTool(const CompilationDatabase &Compilations,
    279                      ArrayRef<std::string> SourcePaths)
    280     : Compilations(Compilations), SourcePaths(SourcePaths),
    281       Files(new FileManager(FileSystemOptions())), DiagConsumer(nullptr) {
    282   appendArgumentsAdjuster(getClangStripOutputAdjuster());
    283   appendArgumentsAdjuster(getClangSyntaxOnlyAdjuster());
    284 }
    285 
    286 ClangTool::~ClangTool() {}
    287 
    288 void ClangTool::mapVirtualFile(StringRef FilePath, StringRef Content) {
    289   MappedFileContents.push_back(std::make_pair(FilePath, Content));
    290 }
    291 
    292 void ClangTool::appendArgumentsAdjuster(ArgumentsAdjuster Adjuster) {
    293   if (ArgsAdjuster)
    294     ArgsAdjuster = combineAdjusters(ArgsAdjuster, Adjuster);
    295   else
    296     ArgsAdjuster = Adjuster;
    297 }
    298 
    299 void ClangTool::clearArgumentsAdjusters() {
    300   ArgsAdjuster = nullptr;
    301 }
    302 
    303 int ClangTool::run(ToolAction *Action) {
    304   // Exists solely for the purpose of lookup of the resource path.
    305   // This just needs to be some symbol in the binary.
    306   static int StaticSymbol;
    307   // The driver detects the builtin header path based on the path of the
    308   // executable.
    309   // FIXME: On linux, GetMainExecutable is independent of the value of the
    310   // first argument, thus allowing ClangTool and runToolOnCode to just
    311   // pass in made-up names here. Make sure this works on other platforms.
    312   std::string MainExecutable =
    313       llvm::sys::fs::getMainExecutable("clang_tool", &StaticSymbol);
    314 
    315   llvm::SmallString<128> InitialDirectory;
    316   if (std::error_code EC = llvm::sys::fs::current_path(InitialDirectory))
    317     llvm::report_fatal_error("Cannot detect current path: " +
    318                              Twine(EC.message()));
    319   bool ProcessingFailed = false;
    320   for (const auto &SourcePath : SourcePaths) {
    321     std::string File(getAbsolutePath(SourcePath));
    322 
    323     // Currently implementations of CompilationDatabase::getCompileCommands can
    324     // change the state of the file system (e.g.  prepare generated headers), so
    325     // this method needs to run right before we invoke the tool, as the next
    326     // file may require a different (incompatible) state of the file system.
    327     //
    328     // FIXME: Make the compilation database interface more explicit about the
    329     // requirements to the order of invocation of its members.
    330     std::vector<CompileCommand> CompileCommandsForFile =
    331         Compilations.getCompileCommands(File);
    332     if (CompileCommandsForFile.empty()) {
    333       // FIXME: There are two use cases here: doing a fuzzy
    334       // "find . -name '*.cc' |xargs tool" match, where as a user I don't care
    335       // about the .cc files that were not found, and the use case where I
    336       // specify all files I want to run over explicitly, where this should
    337       // be an error. We'll want to add an option for this.
    338       llvm::errs() << "Skipping " << File << ". Compile command not found.\n";
    339       continue;
    340     }
    341     for (CompileCommand &CompileCommand : CompileCommandsForFile) {
    342       // FIXME: chdir is thread hostile; on the other hand, creating the same
    343       // behavior as chdir is complex: chdir resolves the path once, thus
    344       // guaranteeing that all subsequent relative path operations work
    345       // on the same path the original chdir resulted in. This makes a
    346       // difference for example on network filesystems, where symlinks might be
    347       // switched during runtime of the tool. Fixing this depends on having a
    348       // file system abstraction that allows openat() style interactions.
    349       if (chdir(CompileCommand.Directory.c_str()))
    350         llvm::report_fatal_error("Cannot chdir into \"" +
    351                                  Twine(CompileCommand.Directory) + "\n!");
    352       std::vector<std::string> CommandLine = CompileCommand.CommandLine;
    353       if (ArgsAdjuster)
    354         CommandLine = ArgsAdjuster(CommandLine);
    355       assert(!CommandLine.empty());
    356       CommandLine[0] = MainExecutable;
    357       // FIXME: We need a callback mechanism for the tool writer to output a
    358       // customized message for each file.
    359       DEBUG({ llvm::dbgs() << "Processing: " << File << ".\n"; });
    360       ToolInvocation Invocation(std::move(CommandLine), Action, Files.get());
    361       Invocation.setDiagnosticConsumer(DiagConsumer);
    362       for (const auto &MappedFile : MappedFileContents)
    363         Invocation.mapVirtualFile(MappedFile.first, MappedFile.second);
    364       if (!Invocation.run()) {
    365         // FIXME: Diagnostics should be used instead.
    366         llvm::errs() << "Error while processing " << File << ".\n";
    367         ProcessingFailed = true;
    368       }
    369       // Return to the initial directory to correctly resolve next file by
    370       // relative path.
    371       if (chdir(InitialDirectory.c_str()))
    372         llvm::report_fatal_error("Cannot chdir into \"" +
    373                                  Twine(InitialDirectory) + "\n!");
    374     }
    375   }
    376   return ProcessingFailed ? 1 : 0;
    377 }
    378 
    379 namespace {
    380 
    381 class ASTBuilderAction : public ToolAction {
    382   std::vector<std::unique_ptr<ASTUnit>> &ASTs;
    383 
    384 public:
    385   ASTBuilderAction(std::vector<std::unique_ptr<ASTUnit>> &ASTs) : ASTs(ASTs) {}
    386 
    387   bool runInvocation(CompilerInvocation *Invocation, FileManager *Files,
    388                      DiagnosticConsumer *DiagConsumer) override {
    389     // FIXME: This should use the provided FileManager.
    390     std::unique_ptr<ASTUnit> AST = ASTUnit::LoadFromCompilerInvocation(
    391         Invocation, CompilerInstance::createDiagnostics(
    392                         &Invocation->getDiagnosticOpts(), DiagConsumer,
    393                         /*ShouldOwnClient=*/false));
    394     if (!AST)
    395       return false;
    396 
    397     ASTs.push_back(std::move(AST));
    398     return true;
    399   }
    400 };
    401 
    402 }
    403 
    404 int ClangTool::buildASTs(std::vector<std::unique_ptr<ASTUnit>> &ASTs) {
    405   ASTBuilderAction Action(ASTs);
    406   return run(&Action);
    407 }
    408 
    409 std::unique_ptr<ASTUnit> buildASTFromCode(const Twine &Code,
    410                                           const Twine &FileName) {
    411   return buildASTFromCodeWithArgs(Code, std::vector<std::string>(), FileName);
    412 }
    413 
    414 std::unique_ptr<ASTUnit>
    415 buildASTFromCodeWithArgs(const Twine &Code,
    416                          const std::vector<std::string> &Args,
    417                          const Twine &FileName) {
    418   SmallString<16> FileNameStorage;
    419   StringRef FileNameRef = FileName.toNullTerminatedStringRef(FileNameStorage);
    420 
    421   std::vector<std::unique_ptr<ASTUnit>> ASTs;
    422   ASTBuilderAction Action(ASTs);
    423   ToolInvocation Invocation(getSyntaxOnlyToolArgs(Args, FileNameRef), &Action,
    424                             nullptr);
    425 
    426   SmallString<1024> CodeStorage;
    427   Invocation.mapVirtualFile(FileNameRef,
    428                             Code.toNullTerminatedStringRef(CodeStorage));
    429   if (!Invocation.run())
    430     return nullptr;
    431 
    432   assert(ASTs.size() == 1);
    433   return std::move(ASTs[0]);
    434 }
    435 
    436 } // end namespace tooling
    437 } // end namespace clang
    438