1 // Copyright 2016 Google Inc. All rights reserved 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // +build ignore 16 17 #include "regen.h" 18 19 #include <sys/stat.h> 20 21 #include <algorithm> 22 #include <memory> 23 #include <mutex> 24 #include <vector> 25 26 #include "fileutil.h" 27 #include "find.h" 28 #include "func.h" 29 #include "io.h" 30 #include "log.h" 31 #include "ninja.h" 32 #include "stats.h" 33 #include "strutil.h" 34 #include "thread_pool.h" 35 36 namespace { 37 38 #define RETURN_TRUE do { \ 39 if (g_flags.dump_kati_stamp) \ 40 needs_regen_ = true; \ 41 else \ 42 return true; \ 43 } while (0) 44 45 bool ShouldIgnoreDirty(StringPiece s) { 46 Pattern pat(g_flags.ignore_dirty_pattern); 47 Pattern nopat(g_flags.no_ignore_dirty_pattern); 48 return pat.Match(s) && !nopat.Match(s); 49 } 50 51 class StampChecker { 52 struct GlobResult { 53 string pat; 54 vector<string> result; 55 }; 56 57 struct ShellResult { 58 CommandOp op; 59 string shell; 60 string shellflag; 61 string cmd; 62 string result; 63 vector<string> missing_dirs; 64 vector<string> files; 65 vector<string> read_dirs; 66 }; 67 68 public: 69 StampChecker() 70 : needs_regen_(false) { 71 } 72 73 ~StampChecker() { 74 for (GlobResult* gr : globs_) { 75 delete gr; 76 } 77 for (ShellResult* sr : commands_) { 78 delete sr; 79 } 80 } 81 82 bool NeedsRegen(double start_time, const string& orig_args) { 83 if (IsMissingOutputs()) 84 RETURN_TRUE; 85 86 if (CheckStep1(orig_args)) 87 RETURN_TRUE; 88 89 if (CheckStep2()) 90 RETURN_TRUE; 91 92 if (!needs_regen_) { 93 FILE* fp = fopen(GetNinjaStampFilename().c_str(), "rb+"); 94 if (!fp) 95 return true; 96 ScopedFile sfp(fp); 97 if (fseek(fp, 0, SEEK_SET) < 0) 98 PERROR("fseek"); 99 size_t r = fwrite(&start_time, sizeof(start_time), 1, fp); 100 CHECK(r == 1); 101 } 102 return needs_regen_; 103 } 104 105 private: 106 bool IsMissingOutputs() { 107 if (!Exists(GetNinjaFilename())) { 108 fprintf(stderr, "%s is missing, regenerating...\n", 109 GetNinjaFilename().c_str()); 110 return true; 111 } 112 if (!Exists(GetNinjaShellScriptFilename())) { 113 fprintf(stderr, "%s is missing, regenerating...\n", 114 GetNinjaShellScriptFilename().c_str()); 115 return true; 116 } 117 return false; 118 } 119 120 bool CheckStep1(const string& orig_args) { 121 #define LOAD_INT(fp) ({ \ 122 int v = LoadInt(fp); \ 123 if (v < 0) { \ 124 fprintf(stderr, "incomplete kati_stamp, regenerating...\n"); \ 125 RETURN_TRUE; \ 126 } \ 127 v; \ 128 }) 129 130 #define LOAD_STRING(fp, s) ({ \ 131 if (!LoadString(fp, s)) { \ 132 fprintf(stderr, "incomplete kati_stamp, regenerating...\n"); \ 133 RETURN_TRUE; \ 134 } \ 135 }) 136 137 const string& stamp_filename = GetNinjaStampFilename(); 138 FILE* fp = fopen(stamp_filename.c_str(), "rb"); 139 if (!fp) { 140 if (g_flags.regen_debug) 141 printf("%s: %s\n", stamp_filename.c_str(), strerror(errno)); 142 return true; 143 } 144 ScopedFile sfp(fp); 145 146 double gen_time; 147 size_t r = fread(&gen_time, sizeof(gen_time), 1, fp); 148 gen_time_ = gen_time; 149 if (r != 1) { 150 fprintf(stderr, "incomplete kati_stamp, regenerating...\n"); 151 RETURN_TRUE; 152 } 153 if (g_flags.regen_debug) 154 printf("Generated time: %f\n", gen_time); 155 156 string s, s2; 157 int num_files = LOAD_INT(fp); 158 for (int i = 0; i < num_files; i++) { 159 LOAD_STRING(fp, &s); 160 double ts = GetTimestamp(s); 161 if (gen_time < ts) { 162 if (g_flags.regen_ignoring_kati_binary) { 163 string kati_binary; 164 GetExecutablePath(&kati_binary); 165 if (s == kati_binary) { 166 fprintf(stderr, "%s was modified, ignored.\n", s.c_str()); 167 continue; 168 } 169 } 170 if (ShouldIgnoreDirty(s)) { 171 if (g_flags.regen_debug) 172 printf("file %s: ignored (%f)\n", s.c_str(), ts); 173 continue; 174 } 175 if (g_flags.dump_kati_stamp) 176 printf("file %s: dirty (%f)\n", s.c_str(), ts); 177 else 178 fprintf(stderr, "%s was modified, regenerating...\n", s.c_str()); 179 RETURN_TRUE; 180 } else if (g_flags.dump_kati_stamp) { 181 printf("file %s: clean (%f)\n", s.c_str(), ts); 182 } 183 } 184 185 int num_undefineds = LOAD_INT(fp); 186 for (int i = 0; i < num_undefineds; i++) { 187 LOAD_STRING(fp, &s); 188 if (getenv(s.c_str())) { 189 if (g_flags.dump_kati_stamp) { 190 printf("env %s: dirty (unset => %s)\n", s.c_str(), getenv(s.c_str())); 191 } else { 192 fprintf(stderr, "Environment variable %s was set, regenerating...\n", 193 s.c_str()); 194 } 195 RETURN_TRUE; 196 } else if (g_flags.dump_kati_stamp) { 197 printf("env %s: clean (unset)\n", s.c_str()); 198 } 199 } 200 201 int num_envs = LOAD_INT(fp); 202 for (int i = 0; i < num_envs; i++) { 203 LOAD_STRING(fp, &s); 204 StringPiece val(getenv(s.c_str())); 205 LOAD_STRING(fp, &s2); 206 if (val != s2) { 207 if (g_flags.dump_kati_stamp) { 208 printf("env %s: dirty (%s => %.*s)\n", 209 s.c_str(), s2.c_str(), SPF(val)); 210 } else { 211 fprintf(stderr, "Environment variable %s was modified (%s => %.*s), " 212 "regenerating...\n", 213 s.c_str(), s2.c_str(), SPF(val)); 214 } 215 RETURN_TRUE; 216 } else if (g_flags.dump_kati_stamp) { 217 printf("env %s: clean (%.*s)\n", s.c_str(), SPF(val)); 218 } 219 } 220 221 int num_globs = LOAD_INT(fp); 222 string pat; 223 for (int i = 0; i < num_globs; i++) { 224 GlobResult* gr = new GlobResult; 225 globs_.push_back(gr); 226 227 LOAD_STRING(fp, &gr->pat); 228 int num_files = LOAD_INT(fp); 229 gr->result.resize(num_files); 230 for (int j = 0; j < num_files; j++) { 231 LOAD_STRING(fp, &gr->result[j]); 232 } 233 } 234 235 int num_crs = LOAD_INT(fp); 236 for (int i = 0; i < num_crs; i++) { 237 ShellResult* sr = new ShellResult; 238 commands_.push_back(sr); 239 sr->op = static_cast<CommandOp>(LOAD_INT(fp)); 240 LOAD_STRING(fp, &sr->shell); 241 LOAD_STRING(fp, &sr->shellflag); 242 LOAD_STRING(fp, &sr->cmd); 243 LOAD_STRING(fp, &sr->result); 244 245 if (sr->op == CommandOp::FIND) { 246 int num_missing_dirs = LOAD_INT(fp); 247 for (int j = 0; j < num_missing_dirs; j++) { 248 LOAD_STRING(fp, &s); 249 sr->missing_dirs.push_back(s); 250 } 251 int num_files = LOAD_INT(fp); 252 for (int j = 0; j < num_files; j++) { 253 LOAD_STRING(fp, &s); 254 sr->files.push_back(s); 255 } 256 int num_read_dirs = LOAD_INT(fp); 257 for (int j = 0; j < num_read_dirs; j++) { 258 LOAD_STRING(fp, &s); 259 sr->read_dirs.push_back(s); 260 } 261 } 262 } 263 264 LoadString(fp, &s); 265 if (orig_args != s) { 266 fprintf(stderr, "arguments changed, regenerating...\n"); 267 RETURN_TRUE; 268 } 269 270 return needs_regen_; 271 } 272 273 bool CheckGlobResult(const GlobResult* gr, string* err) { 274 COLLECT_STATS("glob time (regen)"); 275 vector<string>* files; 276 Glob(gr->pat.c_str(), &files); 277 bool needs_regen = files->size() != gr->result.size(); 278 for (size_t i = 0; i < gr->result.size(); i++) { 279 if (!needs_regen) { 280 if ((*files)[i] != gr->result[i]) { 281 needs_regen = true; 282 break; 283 } 284 } 285 } 286 if (needs_regen) { 287 if (ShouldIgnoreDirty(gr->pat)) { 288 if (g_flags.dump_kati_stamp) { 289 printf("wildcard %s: ignored\n", gr->pat.c_str()); 290 } 291 return false; 292 } 293 if (g_flags.dump_kati_stamp) { 294 printf("wildcard %s: dirty\n", gr->pat.c_str()); 295 } else { 296 *err = StringPrintf("wildcard(%s) was changed, regenerating...\n", 297 gr->pat.c_str()); 298 } 299 } else if (g_flags.dump_kati_stamp) { 300 printf("wildcard %s: clean\n", gr->pat.c_str()); 301 } 302 return needs_regen; 303 } 304 305 bool ShouldRunCommand(const ShellResult* sr) { 306 if (sr->op != CommandOp::FIND) 307 return true; 308 309 COLLECT_STATS("stat time (regen)"); 310 for (const string& dir : sr->missing_dirs) { 311 if (Exists(dir)) 312 return true; 313 } 314 for (const string& file : sr->files) { 315 if (!Exists(file)) 316 return true; 317 } 318 for (const string& dir : sr->read_dirs) { 319 // We assume we rarely do a significant change for the top 320 // directory which affects the results of find command. 321 if (dir == "" || dir == "." || ShouldIgnoreDirty(dir)) 322 continue; 323 324 struct stat st; 325 if (lstat(dir.c_str(), &st) != 0) { 326 return true; 327 } 328 double ts = GetTimestampFromStat(st); 329 if (gen_time_ < ts) { 330 return true; 331 } 332 if (S_ISLNK(st.st_mode)) { 333 ts = GetTimestamp(dir); 334 if (ts < 0 || gen_time_ < ts) 335 return true; 336 } 337 } 338 return false; 339 } 340 341 bool CheckShellResult(const ShellResult* sr, string* err) { 342 if (sr->op == CommandOp::READ_MISSING) { 343 if (Exists(sr->cmd)) { 344 if (g_flags.dump_kati_stamp) 345 printf("file %s: dirty\n", sr->cmd.c_str()); 346 else 347 *err = StringPrintf("$(file <%s) was changed, regenerating...\n", 348 sr->cmd.c_str()); 349 return true; 350 } 351 if (g_flags.dump_kati_stamp) 352 printf("file %s: clean\n", sr->cmd.c_str()); 353 return false; 354 } 355 356 if (sr->op == CommandOp::READ) { 357 double ts = GetTimestamp(sr->cmd); 358 if (gen_time_ < ts) { 359 if (g_flags.dump_kati_stamp) 360 printf("file %s: dirty\n", sr->cmd.c_str()); 361 else 362 *err = StringPrintf("$(file <%s) was changed, regenerating...\n", 363 sr->cmd.c_str()); 364 return true; 365 } 366 if (g_flags.dump_kati_stamp) 367 printf("file %s: clean\n", sr->cmd.c_str()); 368 return false; 369 } 370 371 if (sr->op == CommandOp::WRITE || sr->op == CommandOp::APPEND) { 372 FILE* f = fopen(sr->cmd.c_str(), (sr->op == CommandOp::WRITE) ? "wb" : "ab"); 373 if (f == NULL) { 374 PERROR("fopen"); 375 } 376 377 if (fwrite(&sr->result[0], sr->result.size(), 1, f) != 1) { 378 PERROR("fwrite"); 379 } 380 381 if (fclose(f) != 0) { 382 PERROR("fclose"); 383 } 384 385 if (g_flags.dump_kati_stamp) 386 printf("file %s: clean (write)\n", sr->cmd.c_str()); 387 return false; 388 } 389 390 if (!ShouldRunCommand(sr)) { 391 if (g_flags.regen_debug) 392 printf("shell %s: clean (no rerun)\n", sr->cmd.c_str()); 393 return false; 394 } 395 396 FindCommand fc; 397 if (fc.Parse(sr->cmd) && 398 !fc.chdir.empty() && ShouldIgnoreDirty(fc.chdir)) { 399 if (g_flags.dump_kati_stamp) 400 printf("shell %s: ignored\n", sr->cmd.c_str()); 401 return false; 402 } 403 404 COLLECT_STATS_WITH_SLOW_REPORT("shell time (regen)", sr->cmd.c_str()); 405 string result; 406 RunCommand(sr->shell, sr->shellflag, sr->cmd, RedirectStderr::DEV_NULL, &result); 407 FormatForCommandSubstitution(&result); 408 if (sr->result != result) { 409 if (g_flags.dump_kati_stamp) { 410 printf("shell %s: dirty\n", sr->cmd.c_str()); 411 } else { 412 *err = StringPrintf("$(shell %s) was changed, regenerating...\n", 413 sr->cmd.c_str()); 414 //*err += StringPrintf("%s => %s\n", expected.c_str(), result.c_str()); 415 } 416 return true; 417 } else if (g_flags.regen_debug) { 418 printf("shell %s: clean (rerun)\n", sr->cmd.c_str()); 419 } 420 return false; 421 } 422 423 bool CheckStep2() { 424 unique_ptr<ThreadPool> tp(NewThreadPool(g_flags.num_jobs)); 425 426 tp->Submit([this]() { 427 string err; 428 // TODO: Make glob cache thread safe and create a task for each glob. 429 for (GlobResult* gr : globs_) { 430 if (CheckGlobResult(gr, &err)) { 431 unique_lock<mutex> lock(mu_); 432 if (!needs_regen_) { 433 needs_regen_ = true; 434 msg_ = err; 435 } 436 break; 437 } 438 } 439 }); 440 441 tp->Submit([this]() { 442 for (ShellResult* sr : commands_) { 443 string err; 444 if (CheckShellResult(sr, &err)) { 445 unique_lock<mutex> lock(mu_); 446 if (!needs_regen_) { 447 needs_regen_ = true; 448 msg_ = err; 449 } 450 } 451 } 452 }); 453 454 tp->Wait(); 455 if (needs_regen_) { 456 fprintf(stderr, "%s", msg_.c_str()); 457 } 458 return needs_regen_; 459 } 460 461 private: 462 double gen_time_; 463 vector<GlobResult*> globs_; 464 vector<ShellResult*> commands_; 465 mutex mu_; 466 bool needs_regen_; 467 string msg_; 468 }; 469 470 } // namespace 471 472 bool NeedsRegen(double start_time, const string& orig_args) { 473 return StampChecker().NeedsRegen(start_time, orig_args); 474 } 475