1 // Copyright 2016 Google Inc. All rights reserved 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 #include "regen.h" 16 17 #include <sys/stat.h> 18 19 #include <algorithm> 20 #include <memory> 21 #include <mutex> 22 #include <vector> 23 24 #include "fileutil.h" 25 #include "find.h" 26 #include "io.h" 27 #include "log.h" 28 #include "ninja.h" 29 #include "stats.h" 30 #include "strutil.h" 31 #include "thread_pool.h" 32 33 namespace { 34 35 #define RETURN_TRUE do { \ 36 if (g_flags.dump_kati_stamp) \ 37 needs_regen_ = true; \ 38 else \ 39 return true; \ 40 } while (0) 41 42 bool ShouldIgnoreDirty(StringPiece s) { 43 Pattern pat(g_flags.ignore_dirty_pattern); 44 Pattern nopat(g_flags.no_ignore_dirty_pattern); 45 return pat.Match(s) && !nopat.Match(s); 46 } 47 48 class StampChecker { 49 struct GlobResult { 50 string pat; 51 vector<string> result; 52 }; 53 54 struct ShellResult { 55 string cmd; 56 string result; 57 vector<string> missing_dirs; 58 vector<string> read_dirs; 59 bool has_condition; 60 }; 61 62 public: 63 StampChecker() 64 : needs_regen_(false) { 65 } 66 67 ~StampChecker() { 68 for (GlobResult* gr : globs_) { 69 delete gr; 70 } 71 for (ShellResult* sr : commands_) { 72 delete sr; 73 } 74 } 75 76 bool NeedsRegen(double start_time, const string& orig_args) { 77 if (IsMissingOutputs()) 78 RETURN_TRUE; 79 80 if (CheckStep1(orig_args)) 81 RETURN_TRUE; 82 83 if (CheckStep2()) 84 RETURN_TRUE; 85 86 if (!needs_regen_) { 87 FILE* fp = fopen(GetNinjaStampFilename().c_str(), "rb+"); 88 if (!fp) 89 return true; 90 ScopedFile sfp(fp); 91 if (fseek(fp, 0, SEEK_SET) < 0) 92 PERROR("fseek"); 93 size_t r = fwrite(&start_time, sizeof(start_time), 1, fp); 94 CHECK(r == 1); 95 } 96 return needs_regen_; 97 } 98 99 private: 100 bool IsMissingOutputs() { 101 if (!Exists(GetNinjaFilename())) { 102 fprintf(stderr, "%s is missing, regenerating...\n", 103 GetNinjaFilename().c_str()); 104 return true; 105 } 106 if (!Exists(GetNinjaShellScriptFilename())) { 107 fprintf(stderr, "%s is missing, regenerating...\n", 108 GetNinjaShellScriptFilename().c_str()); 109 return true; 110 } 111 return false; 112 } 113 114 bool CheckStep1(const string& orig_args) { 115 #define LOAD_INT(fp) ({ \ 116 int v = LoadInt(fp); \ 117 if (v < 0) { \ 118 fprintf(stderr, "incomplete kati_stamp, regenerating...\n"); \ 119 RETURN_TRUE; \ 120 } \ 121 v; \ 122 }) 123 124 #define LOAD_STRING(fp, s) ({ \ 125 if (!LoadString(fp, s)) { \ 126 fprintf(stderr, "incomplete kati_stamp, regenerating...\n"); \ 127 RETURN_TRUE; \ 128 } \ 129 }) 130 131 const string& stamp_filename = GetNinjaStampFilename(); 132 FILE* fp = fopen(stamp_filename.c_str(), "rb"); 133 if (!fp) { 134 if (g_flags.dump_kati_stamp) 135 printf("%s: %s\n", stamp_filename.c_str(), strerror(errno)); 136 return true; 137 } 138 ScopedFile sfp(fp); 139 140 double gen_time; 141 size_t r = fread(&gen_time, sizeof(gen_time), 1, fp); 142 gen_time_ = gen_time; 143 if (r != 1) { 144 fprintf(stderr, "incomplete kati_stamp, regenerating...\n"); 145 RETURN_TRUE; 146 } 147 if (g_flags.dump_kati_stamp) 148 printf("Generated time: %f\n", gen_time); 149 150 string s, s2; 151 int num_files = LOAD_INT(fp); 152 for (int i = 0; i < num_files; i++) { 153 LOAD_STRING(fp, &s); 154 double ts = GetTimestamp(s); 155 if (gen_time < ts) { 156 if (g_flags.regen_ignoring_kati_binary) { 157 string kati_binary; 158 GetExecutablePath(&kati_binary); 159 if (s == kati_binary) { 160 fprintf(stderr, "%s was modified, ignored.\n", s.c_str()); 161 continue; 162 } 163 } 164 if (ShouldIgnoreDirty(s)) { 165 if (g_flags.dump_kati_stamp) 166 printf("file %s: ignored (%f)\n", s.c_str(), ts); 167 continue; 168 } 169 if (g_flags.dump_kati_stamp) 170 printf("file %s: dirty (%f)\n", s.c_str(), ts); 171 else 172 fprintf(stderr, "%s was modified, regenerating...\n", s.c_str()); 173 RETURN_TRUE; 174 } else if (g_flags.dump_kati_stamp) { 175 printf("file %s: clean (%f)\n", s.c_str(), ts); 176 } 177 } 178 179 int num_undefineds = LOAD_INT(fp); 180 for (int i = 0; i < num_undefineds; i++) { 181 LOAD_STRING(fp, &s); 182 if (getenv(s.c_str())) { 183 if (g_flags.dump_kati_stamp) { 184 printf("env %s: dirty (unset => %s)\n", s.c_str(), getenv(s.c_str())); 185 } else { 186 fprintf(stderr, "Environment variable %s was set, regenerating...\n", 187 s.c_str()); 188 } 189 RETURN_TRUE; 190 } else if (g_flags.dump_kati_stamp) { 191 printf("env %s: clean (unset)\n", s.c_str()); 192 } 193 } 194 195 int num_envs = LOAD_INT(fp); 196 for (int i = 0; i < num_envs; i++) { 197 LOAD_STRING(fp, &s); 198 StringPiece val(getenv(s.c_str())); 199 LOAD_STRING(fp, &s2); 200 if (val != s2) { 201 if (g_flags.dump_kati_stamp) { 202 printf("env %s: dirty (%s => %.*s)\n", 203 s.c_str(), s2.c_str(), SPF(val)); 204 } else { 205 fprintf(stderr, "Environment variable %s was modified (%s => %.*s), " 206 "regenerating...\n", 207 s.c_str(), s2.c_str(), SPF(val)); 208 } 209 RETURN_TRUE; 210 } else if (g_flags.dump_kati_stamp) { 211 printf("env %s: clean (%.*s)\n", s.c_str(), SPF(val)); 212 } 213 } 214 215 int num_globs = LOAD_INT(fp); 216 string pat; 217 for (int i = 0; i < num_globs; i++) { 218 GlobResult* gr = new GlobResult; 219 globs_.push_back(gr); 220 221 LOAD_STRING(fp, &gr->pat); 222 int num_files = LOAD_INT(fp); 223 gr->result.resize(num_files); 224 for (int j = 0; j < num_files; j++) { 225 LOAD_STRING(fp, &gr->result[j]); 226 } 227 } 228 229 int num_crs = LOAD_INT(fp); 230 for (int i = 0; i < num_crs; i++) { 231 ShellResult* sr = new ShellResult; 232 commands_.push_back(sr); 233 LOAD_STRING(fp, &sr->cmd); 234 LOAD_STRING(fp, &sr->result); 235 sr->has_condition = LOAD_INT(fp); 236 if (!sr->has_condition) 237 continue; 238 239 int num_missing_dirs = LOAD_INT(fp); 240 for (int j = 0; j < num_missing_dirs; j++) { 241 LOAD_STRING(fp, &s); 242 sr->missing_dirs.push_back(s); 243 } 244 int num_read_dirs = LOAD_INT(fp); 245 for (int j = 0; j < num_read_dirs; j++) { 246 LOAD_STRING(fp, &s); 247 sr->read_dirs.push_back(s); 248 } 249 } 250 251 LoadString(fp, &s); 252 if (orig_args != s) { 253 fprintf(stderr, "arguments changed, regenerating...\n"); 254 RETURN_TRUE; 255 } 256 257 return needs_regen_; 258 } 259 260 bool CheckGlobResult(const GlobResult* gr, string* err) { 261 COLLECT_STATS("glob time (regen)"); 262 vector<string>* files; 263 Glob(gr->pat.c_str(), &files); 264 sort(files->begin(), files->end()); 265 bool needs_regen = files->size() != gr->result.size(); 266 for (size_t i = 0; i < gr->result.size(); i++) { 267 if (!needs_regen) { 268 if ((*files)[i] != gr->result[i]) { 269 needs_regen = true; 270 break; 271 } 272 } 273 } 274 if (needs_regen) { 275 if (ShouldIgnoreDirty(gr->pat)) { 276 if (g_flags.dump_kati_stamp) { 277 printf("wildcard %s: ignored\n", gr->pat.c_str()); 278 } 279 return false; 280 } 281 if (g_flags.dump_kati_stamp) { 282 printf("wildcard %s: dirty\n", gr->pat.c_str()); 283 } else { 284 *err = StringPrintf("wildcard(%s) was changed, regenerating...\n", 285 gr->pat.c_str()); 286 } 287 } else if (g_flags.dump_kati_stamp) { 288 printf("wildcard %s: clean\n", gr->pat.c_str()); 289 } 290 return needs_regen; 291 } 292 293 bool ShouldRunCommand(const ShellResult* sr) { 294 if (!sr->has_condition) 295 return true; 296 297 COLLECT_STATS("stat time (regen)"); 298 for (const string& dir : sr->missing_dirs) { 299 if (Exists(dir)) 300 return true; 301 } 302 for (const string& dir : sr->read_dirs) { 303 // We assume we rarely do a significant change for the top 304 // directory which affects the results of find command. 305 if (dir == "" || dir == "." || ShouldIgnoreDirty(dir)) 306 continue; 307 308 struct stat st; 309 if (lstat(dir.c_str(), &st) != 0) { 310 return true; 311 } 312 double ts = GetTimestampFromStat(st); 313 if (gen_time_ < ts) { 314 return true; 315 } 316 if (S_ISLNK(st.st_mode)) { 317 ts = GetTimestamp(dir); 318 if (ts < 0 || gen_time_ < ts) 319 return true; 320 } 321 } 322 return false; 323 } 324 325 bool CheckShellResult(const ShellResult* sr, string* err) { 326 if (!ShouldRunCommand(sr)) { 327 if (g_flags.dump_kati_stamp) 328 printf("shell %s: clean (no rerun)\n", sr->cmd.c_str()); 329 return false; 330 } 331 332 FindCommand fc; 333 if (fc.Parse(sr->cmd) && 334 !fc.chdir.empty() && ShouldIgnoreDirty(fc.chdir)) { 335 if (g_flags.dump_kati_stamp) 336 printf("shell %s: ignored\n", sr->cmd.c_str()); 337 return false; 338 } 339 340 COLLECT_STATS_WITH_SLOW_REPORT("shell time (regen)", sr->cmd.c_str()); 341 string result; 342 RunCommand("/bin/sh", sr->cmd, RedirectStderr::DEV_NULL, &result); 343 FormatForCommandSubstitution(&result); 344 if (sr->result != result) { 345 if (g_flags.dump_kati_stamp) { 346 printf("shell %s: dirty\n", sr->cmd.c_str()); 347 } else { 348 *err = StringPrintf("$(shell %s) was changed, regenerating...\n", 349 sr->cmd.c_str()); 350 //*err += StringPrintf("%s => %s\n", expected.c_str(), result.c_str()); 351 } 352 return true; 353 } else if (g_flags.dump_kati_stamp) { 354 printf("shell %s: clean (rerun)\n", sr->cmd.c_str()); 355 } 356 return false; 357 } 358 359 bool CheckStep2() { 360 unique_ptr<ThreadPool> tp(NewThreadPool(g_flags.num_jobs)); 361 362 tp->Submit([this]() { 363 string err; 364 // TODO: Make glob cache thread safe and create a task for each glob. 365 for (GlobResult* gr : globs_) { 366 if (CheckGlobResult(gr, &err)) { 367 unique_lock<mutex> lock(mu_); 368 if (!needs_regen_) { 369 needs_regen_ = true; 370 msg_ = err; 371 } 372 break; 373 } 374 } 375 }); 376 377 for (ShellResult* sr : commands_) { 378 tp->Submit([this, sr]() { 379 string err; 380 if (CheckShellResult(sr, &err)) { 381 unique_lock<mutex> lock(mu_); 382 if (!needs_regen_) { 383 needs_regen_ = true; 384 msg_ = err; 385 } 386 } 387 }); 388 } 389 390 tp->Wait(); 391 if (needs_regen_) { 392 fprintf(stderr, "%s", msg_.c_str()); 393 } 394 return needs_regen_; 395 } 396 397 private: 398 double gen_time_; 399 vector<GlobResult*> globs_; 400 vector<ShellResult*> commands_; 401 mutex mu_; 402 bool needs_regen_; 403 string msg_; 404 }; 405 406 } // namespace 407 408 bool NeedsRegen(double start_time, const string& orig_args) { 409 return StampChecker().NeedsRegen(start_time, orig_args); 410 } 411
