1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "tools/gn/input_file_manager.h" 6 7 #include "base/bind.h" 8 #include "base/stl_util.h" 9 #include "tools/gn/filesystem_utils.h" 10 #include "tools/gn/parser.h" 11 #include "tools/gn/scheduler.h" 12 #include "tools/gn/scope_per_file_provider.h" 13 #include "tools/gn/tokenizer.h" 14 #include "tools/gn/trace.h" 15 16 namespace { 17 18 void InvokeFileLoadCallback(const InputFileManager::FileLoadCallback& cb, 19 const ParseNode* node) { 20 cb.Run(node); 21 } 22 23 } // namespace 24 25 InputFileManager::InputFileData::InputFileData(const SourceFile& file_name) 26 : file(file_name), 27 loaded(false), 28 sync_invocation(false) { 29 } 30 31 InputFileManager::InputFileData::~InputFileData() { 32 } 33 34 InputFileManager::InputFileManager() { 35 } 36 37 InputFileManager::~InputFileManager() { 38 // Should be single-threaded by now. 39 STLDeleteContainerPairSecondPointers(input_files_.begin(), 40 input_files_.end()); 41 } 42 43 bool InputFileManager::AsyncLoadFile(const LocationRange& origin, 44 const BuildSettings* build_settings, 45 const SourceFile& file_name, 46 const FileLoadCallback& callback, 47 Err* err) { 48 // Try not to schedule callbacks while holding the lock. All cases that don't 49 // want to schedule should return early. Otherwise, this will be scheduled 50 // after we leave the lock. 51 base::Closure schedule_this; 52 { 53 base::AutoLock lock(lock_); 54 55 InputFileMap::const_iterator found = input_files_.find(file_name); 56 if (found == input_files_.end()) { 57 // New file, schedule load. 58 InputFileData* data = new InputFileData(file_name); 59 data->scheduled_callbacks.push_back(callback); 60 input_files_[file_name] = data; 61 62 schedule_this = base::Bind(&InputFileManager::BackgroundLoadFile, 63 this, 64 origin, 65 build_settings, 66 file_name, 67 &data->file); 68 } else { 69 InputFileData* data = found->second; 70 71 // Prevent mixing async and sync loads. See SyncLoadFile for discussion. 72 if (data->sync_invocation) { 73 g_scheduler->FailWithError(Err( 74 origin, "Load type mismatch.", 75 "The file \"" + file_name.value() + "\" was previously loaded\n" 76 "synchronously (via an import) and now you're trying to load it " 77 "asynchronously\n(via a deps rule). This is a class 2 misdemeanor: " 78 "a single input file must\nbe loaded the same way each time to " 79 "avoid blowing my tiny, tiny mind.")); 80 return false; 81 } 82 83 if (data->loaded) { 84 // Can just directly issue the callback on the background thread. 85 schedule_this = base::Bind(&InvokeFileLoadCallback, callback, 86 data->parsed_root.get()); 87 } else { 88 // Load is pending on this file, schedule the invoke. 89 data->scheduled_callbacks.push_back(callback); 90 return true; 91 } 92 } 93 } 94 g_scheduler->pool()->PostWorkerTaskWithShutdownBehavior( 95 FROM_HERE, schedule_this, 96 base::SequencedWorkerPool::BLOCK_SHUTDOWN); 97 return true; 98 } 99 100 const ParseNode* InputFileManager::SyncLoadFile( 101 const LocationRange& origin, 102 const BuildSettings* build_settings, 103 const SourceFile& file_name, 104 Err* err) { 105 base::AutoLock lock(lock_); 106 107 InputFileData* data = NULL; 108 InputFileMap::iterator found = input_files_.find(file_name); 109 if (found == input_files_.end()) { 110 // Haven't seen this file yet, start loading right now. 111 data = new InputFileData(file_name); 112 data->sync_invocation = true; 113 input_files_[file_name] = data; 114 115 base::AutoUnlock unlock(lock_); 116 if (!LoadFile(origin, build_settings, file_name, &data->file, err)) 117 return NULL; 118 } else { 119 // This file has either been loaded or is pending loading. 120 data = found->second; 121 122 if (!data->sync_invocation) { 123 // Don't allow mixing of sync and async loads. If an async load is 124 // scheduled and then a bunch of threads need to load it synchronously 125 // and block on it loading, it could deadlock or at least cause a lot 126 // of wasted CPU while those threads wait for the load to complete (which 127 // may be far back in the input queue). 128 // 129 // We could work around this by promoting the load to a sync load. This 130 // requires a bunch of extra code to either check flags and likely do 131 // extra locking (bad) or to just do both types of load on the file and 132 // deal with the race condition. 133 // 134 // I have no practical way to test this, and generally we should have 135 // all include files processed synchronously and all build files 136 // processed asynchronously, so it doesn't happen in practice. 137 *err = Err( 138 origin, "Load type mismatch.", 139 "The file \"" + file_name.value() + "\" was previously loaded\n" 140 "asynchronously (via a deps rule) and now you're trying to load it " 141 "synchronously.\nThis is a class 2 misdemeanor: a single input file " 142 "must be loaded the same way\neach time to avoid blowing my tiny, " 143 "tiny mind."); 144 return NULL; 145 } 146 147 if (!data->loaded) { 148 // Wait for the already-pending sync load to complete. 149 if (!data->completion_event) 150 data->completion_event.reset(new base::WaitableEvent(false, false)); 151 { 152 base::AutoUnlock unlock(lock_); 153 data->completion_event->Wait(); 154 } 155 // If there were multiple waiters on the same event, we now need to wake 156 // up the next one. 157 data->completion_event->Signal(); 158 } 159 } 160 161 // The other load could have failed. In this case that error will be printed 162 // to the console, but we need to return something here, so make up a 163 // dummy error. 164 if (!data->parsed_root) 165 *err = Err(origin, "File parse failed"); 166 return data->parsed_root.get(); 167 } 168 169 int InputFileManager::GetInputFileCount() const { 170 base::AutoLock lock(lock_); 171 return static_cast<int>(input_files_.size()); 172 } 173 174 void InputFileManager::GetAllPhysicalInputFileNames( 175 std::vector<base::FilePath>* result) const { 176 base::AutoLock lock(lock_); 177 result->reserve(input_files_.size()); 178 for (InputFileMap::const_iterator i = input_files_.begin(); 179 i != input_files_.end(); ++i) { 180 if (!i->second->file.physical_name().empty()) 181 result->push_back(i->second->file.physical_name()); 182 } 183 } 184 185 void InputFileManager::BackgroundLoadFile(const LocationRange& origin, 186 const BuildSettings* build_settings, 187 const SourceFile& name, 188 InputFile* file) { 189 Err err; 190 if (!LoadFile(origin, build_settings, name, file, &err)) 191 g_scheduler->FailWithError(err); 192 } 193 194 bool InputFileManager::LoadFile(const LocationRange& origin, 195 const BuildSettings* build_settings, 196 const SourceFile& name, 197 InputFile* file, 198 Err* err) { 199 // Do all of this stuff outside the lock. We should not give out file 200 // pointers until the read is complete. 201 if (g_scheduler->verbose_logging()) { 202 std::string logmsg = name.value(); 203 if (origin.begin().file()) 204 logmsg += " (referenced from " + origin.begin().Describe(false) + ")"; 205 g_scheduler->Log("Loading", logmsg); 206 } 207 208 // Read. 209 base::FilePath primary_path = build_settings->GetFullPath(name); 210 ScopedTrace load_trace(TraceItem::TRACE_FILE_LOAD, name.value()); 211 if (!file->Load(primary_path)) { 212 if (!build_settings->secondary_source_path().empty()) { 213 // Fall back to secondary source tree. 214 base::FilePath secondary_path = 215 build_settings->GetFullPathSecondary(name); 216 if (!file->Load(secondary_path)) { 217 *err = Err(origin, "Can't load input file.", 218 "Unable to load either \n" + 219 FilePathToUTF8(primary_path) + " or \n" + 220 FilePathToUTF8(secondary_path)); 221 return false; 222 } 223 } else { 224 *err = Err(origin, 225 "Unable to load \"" + FilePathToUTF8(primary_path) + "\"."); 226 return false; 227 } 228 } 229 load_trace.Done(); 230 231 ScopedTrace exec_trace(TraceItem::TRACE_FILE_PARSE, name.value()); 232 233 // Tokenize. 234 std::vector<Token> tokens = Tokenizer::Tokenize(file, err); 235 if (err->has_error()) 236 return false; 237 238 // Parse. 239 scoped_ptr<ParseNode> root = Parser::Parse(tokens, err); 240 if (err->has_error()) 241 return false; 242 ParseNode* unowned_root = root.get(); 243 244 exec_trace.Done(); 245 246 std::vector<FileLoadCallback> callbacks; 247 { 248 base::AutoLock lock(lock_); 249 DCHECK(input_files_.find(name) != input_files_.end()); 250 251 InputFileData* data = input_files_[name]; 252 data->loaded = true; 253 data->tokens.swap(tokens); 254 data->parsed_root = root.Pass(); 255 256 // Unblock waiters on this event. 257 // 258 // It's somewhat bad to signal this inside the lock. When it's used, it's 259 // lazily created inside the lock. So we need to do the check and signal 260 // inside the lock to avoid race conditions on the lazy creation of the 261 // lock. 262 // 263 // We could avoid this by creating the lock every time, but the lock is 264 // very seldom used and will generally be NULL, so my current theory is that 265 // several signals of a completion event inside a lock is better than 266 // creating about 1000 extra locks (one for each file). 267 if (data->completion_event) 268 data->completion_event->Signal(); 269 270 callbacks.swap(data->scheduled_callbacks); 271 } 272 273 // Run pending invocations. Theoretically we could schedule each of these 274 // separately to get some parallelism. But normally there will only be one 275 // item in the list, so that's extra overhead and complexity for no gain. 276 for (size_t i = 0; i < callbacks.size(); i++) 277 callbacks[i].Run(unowned_root); 278 return true; 279 } 280
