Home | History | Annotate | Download | only in gn 
      1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "tools/gn/setup.h"
      6 
      7 #include <stdlib.h>
      8 
      9 #include <algorithm>
     10 #include <sstream>
     11 
     12 #include "base/bind.h"
     13 #include "base/command_line.h"
     14 #include "base/files/file_path.h"
     15 #include "base/files/file_util.h"
     16 #include "base/process/launch.h"
     17 #include "base/strings/string_split.h"
     18 #include "base/strings/string_util.h"
     19 #include "base/strings/utf_string_conversions.h"
     20 #include "build/build_config.h"
     21 #include "tools/gn/commands.h"
     22 #include "tools/gn/filesystem_utils.h"
     23 #include "tools/gn/input_file.h"
     24 #include "tools/gn/parse_tree.h"
     25 #include "tools/gn/parser.h"
     26 #include "tools/gn/source_dir.h"
     27 #include "tools/gn/source_file.h"
     28 #include "tools/gn/standard_out.h"
     29 #include "tools/gn/tokenizer.h"
     30 #include "tools/gn/trace.h"
     31 #include "tools/gn/value.h"
     32 
     33 #if defined(OS_WIN)
     34 #include <windows.h>
     35 #endif
     36 
     37 extern const char kDotfile_Help[] =
     38     ".gn file\n"
     39     "\n"
     40     "  When gn starts, it will search the current directory and parent\n"
     41     "  directories for a file called \".gn\". This indicates the source root.\n"
     42     "  You can override this detection by using the --root command-line\n"
     43     "  argument\n"
     44     "\n"
     45     "  The .gn file in the source root will be executed. The syntax is the\n"
     46     "  same as a buildfile, but with very limited build setup-specific\n"
     47     "  meaning.\n"
     48     "\n"
     49     "  If you specify --root, by default GN will look for the file .gn in\n"
     50     "  that directory. If you want to specify a different file, you can\n"
     51     "  additionally pass --dotfile:\n"
     52     "\n"
     53     "    gn gen out/Debug --root=/home/build --dotfile=/home/my_gn_file.gn\n"
     54     "\n"
     55     "Variables\n"
     56     "\n"
     57     "  buildconfig [required]\n"
     58     "      Label of the build config file. This file will be used to set up\n"
     59     "      the build file execution environment for each toolchain.\n"
     60     "\n"
     61     "  root [optional]\n"
     62     "      Label of the root build target. The GN build will start by loading\n"
     63     "      the build file containing this target name. This defaults to\n"
     64     "      \"//:\" which will cause the file //BUILD.gn to be loaded.\n"
     65     "\n"
     66     "  secondary_source [optional]\n"
     67     "      Label of an alternate directory tree to find input files. When\n"
     68     "      searching for a BUILD.gn file (or the build config file discussed\n"
     69     "      above), the file will first be looked for in the source root.\n"
     70     "      If it's not found, the secondary source root will be checked\n"
     71     "      (which would contain a parallel directory hierarchy).\n"
     72     "\n"
     73     "      This behavior is intended to be used when BUILD.gn files can't be\n"
     74     "      checked in to certain source directories for whatever reason.\n"
     75     "\n"
     76     "      The secondary source root must be inside the main source tree.\n"
     77     "\n"
     78     "Example .gn file contents\n"
     79     "\n"
     80     "  buildconfig = \"//build/config/BUILDCONFIG.gn\"\n"
     81     "\n"
     82     "  root = \"//:root\"\n"
     83     "\n"
     84     "  secondary_source = \"//build/config/temporary_buildfiles/\"\n";
     85 
     86 namespace {
     87 
     88 // More logging.
     89 const char kSwitchVerbose[] = "v";
     90 
     91 // Set build args.
     92 const char kSwitchArgs[] = "args";
     93 
     94 // Set root dir.
     95 const char kSwitchRoot[] = "root";
     96 
     97 // Set dotfile name.
     98 const char kSwitchDotfile[] = "dotfile";
     99 
    100 // Enable timing.
    101 const char kTimeSwitch[] = "time";
    102 
    103 const char kTracelogSwitch[] = "tracelog";
    104 
    105 const base::FilePath::CharType kGnFile[] = FILE_PATH_LITERAL(".gn");
    106 
    107 base::FilePath FindDotFile(const base::FilePath& current_dir) {
    108   base::FilePath try_this_file = current_dir.Append(kGnFile);
    109   if (base::PathExists(try_this_file))
    110     return try_this_file;
    111 
    112   base::FilePath with_no_slash = current_dir.StripTrailingSeparators();
    113   base::FilePath up_one_dir = with_no_slash.DirName();
    114   if (up_one_dir == current_dir)
    115     return base::FilePath();  // Got to the top.
    116 
    117   return FindDotFile(up_one_dir);
    118 }
    119 
    120 // Called on any thread. Post the item to the builder on the main thread.
    121 void ItemDefinedCallback(base::MessageLoop* main_loop,
    122                          scoped_refptr<Builder> builder,
    123                          scoped_ptr<Item> item) {
    124   DCHECK(item);
    125   main_loop->PostTask(FROM_HERE, base::Bind(&Builder::ItemDefined, builder,
    126                                             base::Passed(&item)));
    127 }
    128 
    129 void DecrementWorkCount() {
    130   g_scheduler->DecrementWorkCount();
    131 }
    132 
    133 }  // namespace
    134 
    135 // CommonSetup -----------------------------------------------------------------
    136 
    137 const char CommonSetup::kBuildArgFileName[] = "args.gn";
    138 
    139 CommonSetup::CommonSetup()
    140     : build_settings_(),
    141       loader_(new LoaderImpl(&build_settings_)),
    142       builder_(new Builder(loader_.get())),
    143       root_build_file_("//BUILD.gn"),
    144       check_for_bad_items_(true),
    145       check_for_unused_overrides_(true),
    146       check_public_headers_(false) {
    147   loader_->set_complete_callback(base::Bind(&DecrementWorkCount));
    148 }
    149 
    150 CommonSetup::CommonSetup(const CommonSetup& other)
    151     : build_settings_(other.build_settings_),
    152       loader_(new LoaderImpl(&build_settings_)),
    153       builder_(new Builder(loader_.get())),
    154       root_build_file_(other.root_build_file_),
    155       check_for_bad_items_(other.check_for_bad_items_),
    156       check_for_unused_overrides_(other.check_for_unused_overrides_),
    157       check_public_headers_(other.check_public_headers_) {
    158   loader_->set_complete_callback(base::Bind(&DecrementWorkCount));
    159 }
    160 
    161 CommonSetup::~CommonSetup() {
    162 }
    163 
    164 void CommonSetup::RunPreMessageLoop() {
    165   // Load the root build file.
    166   loader_->Load(root_build_file_, LocationRange(), Label());
    167 
    168   // Will be decremented with the loader is drained.
    169   g_scheduler->IncrementWorkCount();
    170 }
    171 
    172 bool CommonSetup::RunPostMessageLoop() {
    173   Err err;
    174   if (check_for_bad_items_) {
    175     if (!builder_->CheckForBadItems(&err)) {
    176       err.PrintToStdout();
    177       return false;
    178     }
    179   }
    180 
    181   if (check_for_unused_overrides_) {
    182     if (!build_settings_.build_args().VerifyAllOverridesUsed(&err)) {
    183       // TODO(brettw) implement a system of warnings. Until we have a better
    184       // system, print the error but don't return failure.
    185       err.PrintToStdout();
    186       return true;
    187     }
    188   }
    189 
    190   if (check_public_headers_) {
    191     if (!commands::CheckPublicHeaders(&build_settings_,
    192                                       builder_->GetAllResolvedTargets(),
    193                                       std::vector<const Target*>(),
    194                                       false)) {
    195       return false;
    196     }
    197   }
    198 
    199   // Write out tracing and timing if requested.
    200   const CommandLine* cmdline = CommandLine::ForCurrentProcess();
    201   if (cmdline->HasSwitch(kTimeSwitch))
    202     PrintLongHelp(SummarizeTraces());
    203   if (cmdline->HasSwitch(kTracelogSwitch))
    204     SaveTraces(cmdline->GetSwitchValuePath(kTracelogSwitch));
    205 
    206   return true;
    207 }
    208 
    209 // Setup -----------------------------------------------------------------------
    210 
    211 Setup::Setup()
    212     : CommonSetup(),
    213       empty_settings_(&empty_build_settings_, std::string()),
    214       dotfile_scope_(&empty_settings_),
    215       fill_arguments_(true) {
    216   empty_settings_.set_toolchain_label(Label());
    217   build_settings_.set_item_defined_callback(
    218       base::Bind(&ItemDefinedCallback, scheduler_.main_loop(), builder_));
    219 
    220   // The scheduler's main loop wasn't created when the Loader was created, so
    221   // we need to set it now.
    222   loader_->set_main_loop(scheduler_.main_loop());
    223 }
    224 
    225 Setup::~Setup() {
    226 }
    227 
    228 bool Setup::DoSetup(const std::string& build_dir, bool force_create) {
    229   CommandLine* cmdline = CommandLine::ForCurrentProcess();
    230 
    231   scheduler_.set_verbose_logging(cmdline->HasSwitch(kSwitchVerbose));
    232   if (cmdline->HasSwitch(kTimeSwitch) ||
    233       cmdline->HasSwitch(kTracelogSwitch))
    234     EnableTracing();
    235 
    236   ScopedTrace setup_trace(TraceItem::TRACE_SETUP, "DoSetup");
    237 
    238   if (!FillSourceDir(*cmdline))
    239     return false;
    240   if (!RunConfigFile())
    241     return false;
    242   if (!FillOtherConfig(*cmdline))
    243     return false;
    244 
    245   // Must be after FillSourceDir to resolve.
    246   if (!FillBuildDir(build_dir, !force_create))
    247     return false;
    248 
    249   if (fill_arguments_) {
    250     if (!FillArguments(*cmdline))
    251       return false;
    252   }
    253   FillPythonPath();
    254 
    255   return true;
    256 }
    257 
    258 bool Setup::Run() {
    259   RunPreMessageLoop();
    260   if (!scheduler_.Run())
    261     return false;
    262   return RunPostMessageLoop();
    263 }
    264 
    265 Scheduler* Setup::GetScheduler() {
    266   return &scheduler_;
    267 }
    268 
    269 SourceFile Setup::GetBuildArgFile() const {
    270   return SourceFile(build_settings_.build_dir().value() + kBuildArgFileName);
    271 }
    272 
    273 bool Setup::FillArguments(const CommandLine& cmdline) {
    274   // Use the args on the command line if specified, and save them. Do this even
    275   // if the list is empty (this means clear any defaults).
    276   if (cmdline.HasSwitch(kSwitchArgs)) {
    277     if (!FillArgsFromCommandLine(cmdline.GetSwitchValueASCII(kSwitchArgs)))
    278       return false;
    279     SaveArgsToFile();
    280     return true;
    281   }
    282 
    283   // No command line args given, use the arguments from the build dir (if any).
    284   return FillArgsFromFile();
    285 }
    286 
    287 bool Setup::FillArgsFromCommandLine(const std::string& args) {
    288   args_input_file_.reset(new InputFile(SourceFile()));
    289   args_input_file_->SetContents(args);
    290   args_input_file_->set_friendly_name("the command-line \"--args\"");
    291   return FillArgsFromArgsInputFile();
    292 }
    293 
    294 bool Setup::FillArgsFromFile() {
    295   ScopedTrace setup_trace(TraceItem::TRACE_SETUP, "Load args file");
    296 
    297   SourceFile build_arg_source_file = GetBuildArgFile();
    298   base::FilePath build_arg_file =
    299       build_settings_.GetFullPath(build_arg_source_file);
    300 
    301   std::string contents;
    302   if (!base::ReadFileToString(build_arg_file, &contents))
    303     return true;  // File doesn't exist, continue with default args.
    304 
    305   // Add a dependency on the build arguments file. If this changes, we want
    306   // to re-generate the build.
    307   g_scheduler->AddGenDependency(build_arg_file);
    308 
    309   if (contents.empty())
    310     return true;  // Empty file, do nothing.
    311 
    312   args_input_file_.reset(new InputFile(build_arg_source_file));
    313   args_input_file_->SetContents(contents);
    314   args_input_file_->set_friendly_name(
    315       "build arg file (use \"gn args <out_dir>\" to edit)");
    316 
    317   setup_trace.Done();  // Only want to count the load as part of the trace.
    318   return FillArgsFromArgsInputFile();
    319 }
    320 
    321 bool Setup::FillArgsFromArgsInputFile() {
    322   ScopedTrace setup_trace(TraceItem::TRACE_SETUP, "Parse args");
    323 
    324   Err err;
    325   args_tokens_ = Tokenizer::Tokenize(args_input_file_.get(), &err);
    326   if (err.has_error()) {
    327     err.PrintToStdout();
    328     return false;
    329   }
    330 
    331   args_root_ = Parser::Parse(args_tokens_, &err);
    332   if (err.has_error()) {
    333     err.PrintToStdout();
    334     return false;
    335   }
    336 
    337   Scope arg_scope(&empty_settings_);
    338   args_root_->AsBlock()->ExecuteBlockInScope(&arg_scope, &err);
    339   if (err.has_error()) {
    340     err.PrintToStdout();
    341     return false;
    342   }
    343 
    344   // Save the result of the command args.
    345   Scope::KeyValueMap overrides;
    346   arg_scope.GetCurrentScopeValues(&overrides);
    347   build_settings_.build_args().AddArgOverrides(overrides);
    348   return true;
    349 }
    350 
    351 bool Setup::SaveArgsToFile() {
    352   ScopedTrace setup_trace(TraceItem::TRACE_SETUP, "Save args file");
    353 
    354   Scope::KeyValueMap args = build_settings_.build_args().GetAllOverrides();
    355 
    356   std::ostringstream stream;
    357   for (Scope::KeyValueMap::const_iterator i = args.begin();
    358        i != args.end(); ++i) {
    359     stream << i->first.as_string() << " = " << i->second.ToString(true);
    360     stream << std::endl;
    361   }
    362 
    363   // For the first run, the build output dir might not be created yet, so do
    364   // that so we can write a file into it. Ignore errors, we'll catch the error
    365   // when we try to write a file to it below.
    366   base::FilePath build_arg_file =
    367       build_settings_.GetFullPath(GetBuildArgFile());
    368   base::CreateDirectory(build_arg_file.DirName());
    369 
    370   std::string contents = stream.str();
    371 #if defined(OS_WIN)
    372   // Use Windows lineendings for this file since it will often open in
    373   // Notepad which can't handle Unix ones.
    374   ReplaceSubstringsAfterOffset(&contents, 0, "\n", "\r\n");
    375 #endif
    376   if (base::WriteFile(build_arg_file, contents.c_str(),
    377       static_cast<int>(contents.size())) == -1) {
    378     Err(Location(), "Args file could not be written.",
    379       "The file is \"" + FilePathToUTF8(build_arg_file) +
    380         "\"").PrintToStdout();
    381     return false;
    382   }
    383 
    384   // Add a dependency on the build arguments file. If this changes, we want
    385   // to re-generate the build.
    386   g_scheduler->AddGenDependency(build_arg_file);
    387 
    388   return true;
    389 }
    390 
    391 bool Setup::FillSourceDir(const CommandLine& cmdline) {
    392   // Find the .gn file.
    393   base::FilePath root_path;
    394 
    395   // Prefer the command line args to the config file.
    396   base::FilePath relative_root_path = cmdline.GetSwitchValuePath(kSwitchRoot);
    397   if (!relative_root_path.empty()) {
    398     root_path = base::MakeAbsoluteFilePath(relative_root_path);
    399     if (root_path.empty()) {
    400       Err(Location(), "Root source path not found.",
    401           "The path \"" + FilePathToUTF8(relative_root_path) +
    402           "\" doesn't exist.").PrintToStdout();
    403       return false;
    404     }
    405 
    406     // When --root is specified, an alternate --dotfile can also be set.
    407     // --dotfile should be a real file path and not a "//foo" source-relative
    408     // path.
    409     base::FilePath dot_file_path = cmdline.GetSwitchValuePath(kSwitchDotfile);
    410     if (dot_file_path.empty()) {
    411       dotfile_name_ = root_path.Append(kGnFile);
    412     } else {
    413       dotfile_name_ = base::MakeAbsoluteFilePath(dot_file_path);
    414       if (dotfile_name_.empty()) {
    415         Err(Location(), "Could not load dotfile.",
    416             "The file \"" + FilePathToUTF8(dot_file_path) +
    417             "\" cound't be loaded.").PrintToStdout();
    418         return false;
    419       }
    420     }
    421   } else {
    422     // In the default case, look for a dotfile and that also tells us where the
    423     // source root is.
    424     base::FilePath cur_dir;
    425     base::GetCurrentDirectory(&cur_dir);
    426     dotfile_name_ = FindDotFile(cur_dir);
    427     if (dotfile_name_.empty()) {
    428       Err(Location(), "Can't find source root.",
    429           "I could not find a \".gn\" file in the current directory or any "
    430           "parent,\nand the --root command-line argument was not specified.")
    431           .PrintToStdout();
    432       return false;
    433     }
    434     root_path = dotfile_name_.DirName();
    435   }
    436 
    437   if (scheduler_.verbose_logging())
    438     scheduler_.Log("Using source root", FilePathToUTF8(root_path));
    439   build_settings_.SetRootPath(root_path);
    440 
    441   return true;
    442 }
    443 
    444 bool Setup::FillBuildDir(const std::string& build_dir, bool require_exists) {
    445   SourceDir resolved =
    446       SourceDirForCurrentDirectory(build_settings_.root_path()).
    447           ResolveRelativeDir(build_dir);
    448   if (resolved.is_null()) {
    449     Err(Location(), "Couldn't resolve build directory.",
    450         "The build directory supplied (\"" + build_dir + "\") was not valid.").
    451         PrintToStdout();
    452     return false;
    453   }
    454 
    455   if (scheduler_.verbose_logging())
    456     scheduler_.Log("Using build dir", resolved.value());
    457 
    458   if (require_exists) {
    459     base::FilePath build_dir_path = build_settings_.GetFullPath(resolved);
    460     if (!base::PathExists(build_dir_path.Append(
    461             FILE_PATH_LITERAL("build.ninja")))) {
    462       Err(Location(), "Not a build directory.",
    463           "This command requires an existing build directory. I interpreted "
    464           "your input\n\"" + build_dir + "\" as:\n  " +
    465           FilePathToUTF8(build_dir_path) +
    466           "\nwhich doesn't seem to contain a previously-generated build.")
    467           .PrintToStdout();
    468       return false;
    469     }
    470   }
    471 
    472   build_settings_.SetBuildDir(resolved);
    473   return true;
    474 }
    475 
    476 void Setup::FillPythonPath() {
    477   // Trace this since it tends to be a bit slow on Windows.
    478   ScopedTrace setup_trace(TraceItem::TRACE_SETUP, "Fill Python Path");
    479 #if defined(OS_WIN)
    480   // Find Python on the path so we can use the absolute path in the build.
    481   const base::char16 kGetPython[] =
    482       L"cmd.exe /c python -c \"import sys; print sys.executable\"";
    483   std::string python_path;
    484   if (base::GetAppOutput(kGetPython, &python_path)) {
    485     base::TrimWhitespaceASCII(python_path, base::TRIM_ALL, &python_path);
    486     if (scheduler_.verbose_logging())
    487       scheduler_.Log("Found python", python_path);
    488   } else {
    489     scheduler_.Log("WARNING", "Could not find python on path, using "
    490         "just \"python.exe\"");
    491     python_path = "python.exe";
    492   }
    493   build_settings_.set_python_path(base::FilePath(base::UTF8ToUTF16(python_path))
    494                                       .NormalizePathSeparatorsTo('/'));
    495 #else
    496   build_settings_.set_python_path(base::FilePath("python"));
    497 #endif
    498 }
    499 
    500 bool Setup::RunConfigFile() {
    501   if (scheduler_.verbose_logging())
    502     scheduler_.Log("Got dotfile", FilePathToUTF8(dotfile_name_));
    503 
    504   dotfile_input_file_.reset(new InputFile(SourceFile("//.gn")));
    505   if (!dotfile_input_file_->Load(dotfile_name_)) {
    506     Err(Location(), "Could not load dotfile.",
    507         "The file \"" + FilePathToUTF8(dotfile_name_) + "\" cound't be loaded")
    508         .PrintToStdout();
    509     return false;
    510   }
    511 
    512   Err err;
    513   dotfile_tokens_ = Tokenizer::Tokenize(dotfile_input_file_.get(), &err);
    514   if (err.has_error()) {
    515     err.PrintToStdout();
    516     return false;
    517   }
    518 
    519   dotfile_root_ = Parser::Parse(dotfile_tokens_, &err);
    520   if (err.has_error()) {
    521     err.PrintToStdout();
    522     return false;
    523   }
    524 
    525   dotfile_root_->AsBlock()->ExecuteBlockInScope(&dotfile_scope_, &err);
    526   if (err.has_error()) {
    527     err.PrintToStdout();
    528     return false;
    529   }
    530 
    531   return true;
    532 }
    533 
    534 bool Setup::FillOtherConfig(const CommandLine& cmdline) {
    535   Err err;
    536 
    537   // Secondary source path, read from the config file if present.
    538   // Read from the config file if present.
    539   const Value* secondary_value =
    540       dotfile_scope_.GetValue("secondary_source", true);
    541   if (secondary_value) {
    542     if (!secondary_value->VerifyTypeIs(Value::STRING, &err)) {
    543       err.PrintToStdout();
    544       return false;
    545     }
    546     build_settings_.SetSecondarySourcePath(
    547         SourceDir(secondary_value->string_value()));
    548   }
    549 
    550   // Root build file.
    551   const Value* root_value = dotfile_scope_.GetValue("root", true);
    552   if (root_value) {
    553     if (!root_value->VerifyTypeIs(Value::STRING, &err)) {
    554       err.PrintToStdout();
    555       return false;
    556     }
    557 
    558     Label root_target_label =
    559         Label::Resolve(SourceDir("//"), Label(), *root_value, &err);
    560     if (err.has_error()) {
    561       err.PrintToStdout();
    562       return false;
    563     }
    564 
    565     root_build_file_ = Loader::BuildFileForLabel(root_target_label);
    566   }
    567 
    568   // Build config file.
    569   const Value* build_config_value =
    570       dotfile_scope_.GetValue("buildconfig", true);
    571   if (!build_config_value) {
    572     Err(Location(), "No build config file.",
    573         "Your .gn file (\"" + FilePathToUTF8(dotfile_name_) + "\")\n"
    574         "didn't specify a \"buildconfig\" value.").PrintToStdout();
    575     return false;
    576   } else if (!build_config_value->VerifyTypeIs(Value::STRING, &err)) {
    577     err.PrintToStdout();
    578     return false;
    579   }
    580   build_settings_.set_build_config_file(
    581       SourceFile(build_config_value->string_value()));
    582 
    583   return true;
    584 }
    585 
    586 // DependentSetup --------------------------------------------------------------
    587 
    588 DependentSetup::DependentSetup(Setup* derive_from)
    589     : CommonSetup(*derive_from),
    590       scheduler_(derive_from->GetScheduler()) {
    591   build_settings_.set_item_defined_callback(
    592       base::Bind(&ItemDefinedCallback, scheduler_->main_loop(), builder_));
    593 }
    594 
    595 DependentSetup::DependentSetup(DependentSetup* derive_from)
    596     : CommonSetup(*derive_from),
    597       scheduler_(derive_from->GetScheduler()) {
    598   build_settings_.set_item_defined_callback(
    599       base::Bind(&ItemDefinedCallback, scheduler_->main_loop(), builder_));
    600 }
    601 
    602 DependentSetup::~DependentSetup() {
    603 }
    604 
    605 Scheduler* DependentSetup::GetScheduler() {
    606   return scheduler_;
    607 }
    608 
    609 void DependentSetup::RunPreMessageLoop() {
    610   CommonSetup::RunPreMessageLoop();
    611 }
    612 
    613 bool DependentSetup::RunPostMessageLoop() {
    614   return CommonSetup::RunPostMessageLoop();
    615 }
    616 
    617