Home | History | Annotate | Download | only in gn
      1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "base/command_line.h"
      6 #include "base/file_util.h"
      7 #include "base/logging.h"
      8 #include "base/process/kill.h"
      9 #include "base/process/launch.h"
     10 #include "base/strings/string_number_conversions.h"
     11 #include "base/strings/utf_string_conversions.h"
     12 #include "base/time/time.h"
     13 #include "build/build_config.h"
     14 #include "tools/gn/err.h"
     15 #include "tools/gn/filesystem_utils.h"
     16 #include "tools/gn/functions.h"
     17 #include "tools/gn/input_conversion.h"
     18 #include "tools/gn/input_file.h"
     19 #include "tools/gn/parse_tree.h"
     20 #include "tools/gn/scheduler.h"
     21 #include "tools/gn/trace.h"
     22 #include "tools/gn/value.h"
     23 
     24 #if defined(OS_WIN)
     25 #include <windows.h>
     26 
     27 #include "base/win/scoped_handle.h"
     28 #include "base/win/scoped_process_information.h"
     29 #endif
     30 
     31 #if defined(OS_POSIX)
     32 #include <fcntl.h>
     33 #include <unistd.h>
     34 
     35 #include "base/posix/file_descriptor_shuffle.h"
     36 #endif
     37 
     38 namespace functions {
     39 
     40 namespace {
     41 
     42 const char kNoExecSwitch[] = "no-exec";
     43 
     44 #if defined(OS_WIN)
     45 bool ExecProcess(const CommandLine& cmdline,
     46                  const base::FilePath& startup_dir,
     47                  std::string* std_out,
     48                  std::string* std_err,
     49                  int* exit_code) {
     50   SECURITY_ATTRIBUTES sa_attr;
     51   // Set the bInheritHandle flag so pipe handles are inherited.
     52   sa_attr.nLength = sizeof(SECURITY_ATTRIBUTES);
     53   sa_attr.bInheritHandle = TRUE;
     54   sa_attr.lpSecurityDescriptor = NULL;
     55 
     56   // Create the pipe for the child process's STDOUT.
     57   HANDLE out_read = NULL;
     58   HANDLE out_write = NULL;
     59   if (!CreatePipe(&out_read, &out_write, &sa_attr, 0)) {
     60     NOTREACHED() << "Failed to create pipe";
     61     return false;
     62   }
     63   base::win::ScopedHandle scoped_out_read(out_read);
     64   base::win::ScopedHandle scoped_out_write(out_write);
     65 
     66   // Create the pipe for the child process's STDERR.
     67   HANDLE err_read = NULL;
     68   HANDLE err_write = NULL;
     69   if (!CreatePipe(&err_read, &err_write, &sa_attr, 0)) {
     70     NOTREACHED() << "Failed to create pipe";
     71     return false;
     72   }
     73   base::win::ScopedHandle scoped_err_read(err_read);
     74   base::win::ScopedHandle scoped_err_write(err_write);
     75 
     76   // Ensure the read handle to the pipe for STDOUT/STDERR is not inherited.
     77   if (!SetHandleInformation(out_read, HANDLE_FLAG_INHERIT, 0)) {
     78     NOTREACHED() << "Failed to disabled pipe inheritance";
     79     return false;
     80   }
     81   if (!SetHandleInformation(err_read, HANDLE_FLAG_INHERIT, 0)) {
     82     NOTREACHED() << "Failed to disabled pipe inheritance";
     83     return false;
     84   }
     85 
     86   base::FilePath::StringType cmdline_str(cmdline.GetCommandLineString());
     87 
     88   STARTUPINFO start_info = {};
     89 
     90   start_info.cb = sizeof(STARTUPINFO);
     91   start_info.hStdOutput = out_write;
     92   // Keep the normal stdin.
     93   start_info.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
     94   // FIXME(brettw) set stderr here when we actually read it below.
     95   //start_info.hStdError = err_write;
     96   start_info.hStdError = GetStdHandle(STD_ERROR_HANDLE);
     97   start_info.dwFlags |= STARTF_USESTDHANDLES;
     98 
     99   // Create the child process.
    100   PROCESS_INFORMATION temp_process_info = {};
    101   if (!CreateProcess(NULL,
    102                      &cmdline_str[0],
    103                      NULL, NULL,
    104                      TRUE,  // Handles are inherited.
    105                      0, NULL,
    106                      startup_dir.value().c_str(),
    107                      &start_info, &temp_process_info)) {
    108     return false;
    109   }
    110   base::win::ScopedProcessInformation proc_info(temp_process_info);
    111 
    112   // Close our writing end of pipes now. Otherwise later read would not be able
    113   // to detect end of child's output.
    114   scoped_out_write.Close();
    115   scoped_err_write.Close();
    116 
    117   // Read output from the child process's pipe for STDOUT
    118   const int kBufferSize = 1024;
    119   char buffer[kBufferSize];
    120 
    121   // FIXME(brettw) read from stderr here! This is complicated because we want
    122   // to read both of them at the same time, probably need overlapped I/O.
    123   // Also uncomment start_info code above.
    124   for (;;) {
    125     DWORD bytes_read = 0;
    126     BOOL success = ReadFile(out_read, buffer, kBufferSize, &bytes_read, NULL);
    127     if (!success || bytes_read == 0)
    128       break;
    129     std_out->append(buffer, bytes_read);
    130   }
    131 
    132   // Let's wait for the process to finish.
    133   WaitForSingleObject(proc_info.process_handle(), INFINITE);
    134 
    135   DWORD dw_exit_code;
    136   GetExitCodeProcess(proc_info.process_handle(), &dw_exit_code);
    137   *exit_code = static_cast<int>(dw_exit_code);
    138 
    139   return true;
    140 }
    141 #else
    142 bool ExecProcess(const CommandLine& cmdline,
    143                  const base::FilePath& startup_dir,
    144                  std::string* std_out,
    145                  std::string* std_err,
    146                  int* exit_code) {
    147   *exit_code = EXIT_FAILURE;
    148 
    149   std::vector<std::string> argv = cmdline.argv();
    150 
    151   int pipe_fd[2];
    152   pid_t pid;
    153   base::InjectiveMultimap fd_shuffle1, fd_shuffle2;
    154   scoped_ptr<char*[]> argv_cstr(new char*[argv.size() + 1]);
    155 
    156   fd_shuffle1.reserve(3);
    157   fd_shuffle2.reserve(3);
    158 
    159   if (pipe(pipe_fd) < 0)
    160     return false;
    161 
    162   switch (pid = fork()) {
    163     case -1:  // error
    164       close(pipe_fd[0]);
    165       close(pipe_fd[1]);
    166       return false;
    167     case 0:  // child
    168       {
    169         // DANGER: no calls to malloc are allowed from now on:
    170         // http://crbug.com/36678
    171 
    172         // Obscure fork() rule: in the child, if you don't end up doing exec*(),
    173         // you call _exit() instead of exit(). This is because _exit() does not
    174         // call any previously-registered (in the parent) exit handlers, which
    175         // might do things like block waiting for threads that don't even exist
    176         // in the child.
    177         int dev_null = open("/dev/null", O_WRONLY);
    178         if (dev_null < 0)
    179           _exit(127);
    180 
    181         fd_shuffle1.push_back(
    182             base::InjectionArc(pipe_fd[1], STDOUT_FILENO, true));
    183         fd_shuffle1.push_back(
    184             base::InjectionArc(dev_null, STDERR_FILENO, true));
    185         fd_shuffle1.push_back(
    186             base::InjectionArc(dev_null, STDIN_FILENO, true));
    187         // Adding another element here? Remeber to increase the argument to
    188         // reserve(), above.
    189 
    190         std::copy(fd_shuffle1.begin(), fd_shuffle1.end(),
    191                   std::back_inserter(fd_shuffle2));
    192 
    193         if (!ShuffleFileDescriptors(&fd_shuffle1))
    194           _exit(127);
    195 
    196         file_util::SetCurrentDirectory(startup_dir);
    197 
    198         // TODO(brettw) the base version GetAppOutput does a
    199         // CloseSuperfluousFds call here. Do we need this?
    200 
    201         for (size_t i = 0; i < argv.size(); i++)
    202           argv_cstr[i] = const_cast<char*>(argv[i].c_str());
    203         argv_cstr[argv.size()] = NULL;
    204         execvp(argv_cstr[0], argv_cstr.get());
    205         _exit(127);
    206       }
    207     default:  // parent
    208       {
    209         // Close our writing end of pipe now. Otherwise later read would not
    210         // be able to detect end of child's output (in theory we could still
    211         // write to the pipe).
    212         close(pipe_fd[1]);
    213 
    214         char buffer[256];
    215         ssize_t bytes_read = 0;
    216 
    217         while (true) {
    218           bytes_read = HANDLE_EINTR(read(pipe_fd[0], buffer, sizeof(buffer)));
    219           if (bytes_read <= 0)
    220             break;
    221           std_out->append(buffer, bytes_read);
    222         }
    223         close(pipe_fd[0]);
    224 
    225         return base::WaitForExitCode(pid, exit_code);
    226       }
    227   }
    228 
    229   return false;
    230 }
    231 #endif
    232 
    233 }  // namespace
    234 
    235 const char kExecScript[] = "exec_script";
    236 const char kExecScript_Help[] =
    237     "exec_script: Synchronously run a script and return the output.\n"
    238     "\n"
    239     "  exec_script(filename, arguments, input_conversion,\n"
    240     "              [file_dependencies])\n"
    241     "\n"
    242     "  Runs the given script, returning the stdout of the script. The build\n"
    243     "  generation will fail if the script does not exist or returns a nonzero\n"
    244     "  exit code.\n"
    245     "\n"
    246     "  The current directory when executing the script will be the root\n"
    247     "  build directory. If you are passing file names, you will want to use\n"
    248     "  the to_build_dir() function to make file names relative to this\n"
    249     "  path (see \"gn help to_build_dir\").\n"
    250     "\n"
    251     "Arguments:\n"
    252     "\n"
    253     "  filename:\n"
    254     "      File name of python script to execute. Non-absolute names will\n"
    255     "      be treated as relative to the current build file.\n"
    256     "\n"
    257     "  arguments:\n"
    258     "      A list of strings to be passed to the script as arguments.\n"
    259     "\n"
    260     "  input_conversion:\n"
    261     "      Controls how the file is read and parsed.\n"
    262     "      See \"gn help input_conversion\".\n"
    263     "\n"
    264     "  dependencies:\n"
    265     "      (Optional) A list of files that this script reads or otherwise\n"
    266     "      depends on. These dependencies will be added to the build result\n"
    267     "      such that if any of them change, the build will be regenerated and\n"
    268     "      the script will be re-run.\n"
    269     "\n"
    270     "      The script itself will be an implicit dependency so you do not\n"
    271     "      need to list it.\n"
    272     "\n"
    273     "Example:\n"
    274     "\n"
    275     "  all_lines = exec_script(\"myscript.py\", [some_input], \"list lines\",\n"
    276     "                          [ to_build_dir(\"data_file.txt\") ])\n";
    277 
    278 Value RunExecScript(Scope* scope,
    279                     const FunctionCallNode* function,
    280                     const std::vector<Value>& args,
    281                     Err* err) {
    282   if (args.size() != 3 && args.size() != 4) {
    283     *err = Err(function->function(), "Wrong number of args to write_file",
    284                "I expected three or four arguments.");
    285     return Value();
    286   }
    287 
    288   const Settings* settings = scope->settings();
    289   const BuildSettings* build_settings = settings->build_settings();
    290   const SourceDir& cur_dir = scope->GetSourceDir();
    291 
    292   // Find the python script to run.
    293   if (!args[0].VerifyTypeIs(Value::STRING, err))
    294     return Value();
    295   SourceFile script_source =
    296       cur_dir.ResolveRelativeFile(args[0].string_value());
    297   base::FilePath script_path = build_settings->GetFullPath(script_source);
    298   if (!build_settings->secondary_source_path().empty() &&
    299       !base::PathExists(script_path)) {
    300     // Fall back to secondary source root when the file doesn't exist.
    301     script_path = build_settings->GetFullPathSecondary(script_source);
    302   }
    303 
    304   ScopedTrace trace(TraceItem::TRACE_SCRIPT_EXECUTE, script_source.value());
    305   trace.SetToolchain(settings->toolchain_label());
    306 
    307   // Add all dependencies of this script, including the script itself, to the
    308   // build deps.
    309   g_scheduler->AddGenDependency(script_path);
    310   if (args.size() == 4) {
    311     const Value& deps_value = args[3];
    312     if (!deps_value.VerifyTypeIs(Value::LIST, err))
    313       return Value();
    314 
    315     for (size_t i = 0; i < deps_value.list_value().size(); i++) {
    316       if (!deps_value.list_value()[0].VerifyTypeIs(Value::STRING, err))
    317         return Value();
    318       g_scheduler->AddGenDependency(
    319           build_settings->GetFullPath(cur_dir.ResolveRelativeFile(
    320               deps_value.list_value()[0].string_value())));
    321     }
    322   }
    323 
    324   // Make the command line.
    325   const base::FilePath& python_path = build_settings->python_path();
    326   CommandLine cmdline(python_path);
    327   cmdline.AppendArgPath(script_path);
    328 
    329   const Value& script_args = args[1];
    330   if (!script_args.VerifyTypeIs(Value::LIST, err))
    331     return Value();
    332   for (size_t i = 0; i < script_args.list_value().size(); i++) {
    333     if (!script_args.list_value()[i].VerifyTypeIs(Value::STRING, err))
    334       return Value();
    335     cmdline.AppendArg(script_args.list_value()[i].string_value());
    336   }
    337 
    338   // Log command line for debugging help.
    339   trace.SetCommandLine(cmdline);
    340   base::TimeTicks begin_exec;
    341   if (g_scheduler->verbose_logging()) {
    342 #if defined(OS_WIN)
    343     g_scheduler->Log("Pythoning", UTF16ToUTF8(cmdline.GetCommandLineString()));
    344 #else
    345     g_scheduler->Log("Pythoning", cmdline.GetCommandLineString());
    346 #endif
    347     begin_exec = base::TimeTicks::Now();
    348   }
    349 
    350   base::FilePath startup_dir =
    351       build_settings->GetFullPath(build_settings->build_dir());
    352   // The first time a build is run, no targets will have been written so the
    353   // build output directory won't exist. We need to make sure it does before
    354   // running any scripts with this as its startup directory, although it will
    355   // be relatively rare that the directory won't exist by the time we get here.
    356   //
    357   // If this shows up on benchmarks, we can cache whether we've done this
    358   // or not and skip creating the directory.
    359   base::CreateDirectory(startup_dir);
    360 
    361   // Execute the process.
    362   // TODO(brettw) set the environment block.
    363   std::string output;
    364   std::string stderr_output;  // TODO(brettw) not hooked up, see above.
    365   int exit_code = 0;
    366   if (!CommandLine::ForCurrentProcess()->HasSwitch(kNoExecSwitch)) {
    367     if (!ExecProcess(cmdline, startup_dir,
    368                      &output, &stderr_output, &exit_code)) {
    369       *err = Err(function->function(), "Could not execute python.",
    370           "I was trying to execute \"" + FilePathToUTF8(python_path) + "\".");
    371       return Value();
    372     }
    373   }
    374   if (g_scheduler->verbose_logging()) {
    375     g_scheduler->Log("Pythoning", script_source.value() + " took " +
    376         base::Int64ToString(
    377             (base::TimeTicks::Now() - begin_exec).InMilliseconds()) +
    378         "ms");
    379   }
    380 
    381   // TODO(brettw) maybe we need stderr also for reasonable stack dumps.
    382   if (exit_code != 0) {
    383     std::string msg = "Current dir: " + FilePathToUTF8(startup_dir) +
    384         "\nCommand: " + FilePathToUTF8(cmdline.GetCommandLineString()) +
    385         "\nReturned " + base::IntToString(exit_code);
    386     if (!output.empty())
    387       msg += " and printed out:\n\n" + output;
    388     else
    389       msg += ".";
    390     *err = Err(function->function(), "Script returned non-zero exit code.",
    391                msg);
    392     return Value();
    393   }
    394 
    395   return ConvertInputToValue(output, function, args[2], err);
    396 }
    397 
    398 }  // namespace functions
    399