Home | History | Annotate | Download | only in kati 
      1 // Copyright 2016 Google Inc. All rights reserved
      2 //
      3 // Licensed under the Apache License, Version 2.0 (the "License");
      4 // you may not use this file except in compliance with the License.
      5 // You may obtain a copy of the License at
      6 //
      7 //      http://www.apache.org/licenses/LICENSE-2.0
      8 //
      9 // Unless required by applicable law or agreed to in writing, software
     10 // distributed under the License is distributed on an "AS IS" BASIS,
     11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     12 // See the License for the specific language governing permissions and
     13 // limitations under the License.
     14 
     15 // +build ignore
     16 
     17 #include "regen.h"
     18 
     19 #include <sys/stat.h>
     20 
     21 #include <algorithm>
     22 #include <memory>
     23 #include <mutex>
     24 #include <vector>
     25 
     26 #include "fileutil.h"
     27 #include "find.h"
     28 #include "func.h"
     29 #include "io.h"
     30 #include "log.h"
     31 #include "ninja.h"
     32 #include "stats.h"
     33 #include "strutil.h"
     34 #include "thread_pool.h"
     35 
     36 namespace {
     37 
     38 #define RETURN_TRUE do {                        \
     39       if (g_flags.dump_kati_stamp)              \
     40         needs_regen_ = true;                    \
     41       else                                      \
     42         return true;                            \
     43     } while (0)
     44 
     45 bool ShouldIgnoreDirty(StringPiece s) {
     46   Pattern pat(g_flags.ignore_dirty_pattern);
     47   Pattern nopat(g_flags.no_ignore_dirty_pattern);
     48   return pat.Match(s) && !nopat.Match(s);
     49 }
     50 
     51 class StampChecker {
     52   struct GlobResult {
     53     string pat;
     54     vector<string> result;
     55   };
     56 
     57   struct ShellResult {
     58     CommandOp op;
     59     string shell;
     60     string shellflag;
     61     string cmd;
     62     string result;
     63     vector<string> missing_dirs;
     64     vector<string> files;
     65     vector<string> read_dirs;
     66   };
     67 
     68  public:
     69   StampChecker()
     70       : needs_regen_(false) {
     71   }
     72 
     73   ~StampChecker() {
     74     for (GlobResult* gr : globs_) {
     75       delete gr;
     76     }
     77     for (ShellResult* sr : commands_) {
     78       delete sr;
     79     }
     80   }
     81 
     82   bool NeedsRegen(double start_time, const string& orig_args) {
     83     if (IsMissingOutputs())
     84       RETURN_TRUE;
     85 
     86     if (CheckStep1(orig_args))
     87       RETURN_TRUE;
     88 
     89     if (CheckStep2())
     90       RETURN_TRUE;
     91 
     92     if (!needs_regen_) {
     93       FILE* fp = fopen(GetNinjaStampFilename().c_str(), "rb+");
     94       if (!fp)
     95         return true;
     96       ScopedFile sfp(fp);
     97       if (fseek(fp, 0, SEEK_SET) < 0)
     98         PERROR("fseek");
     99       size_t r = fwrite(&start_time, sizeof(start_time), 1, fp);
    100       CHECK(r == 1);
    101     }
    102     return needs_regen_;
    103   }
    104 
    105  private:
    106   bool IsMissingOutputs() {
    107     if (!Exists(GetNinjaFilename())) {
    108       fprintf(stderr, "%s is missing, regenerating...\n",
    109               GetNinjaFilename().c_str());
    110       return true;
    111     }
    112     if (!Exists(GetNinjaShellScriptFilename())) {
    113       fprintf(stderr, "%s is missing, regenerating...\n",
    114               GetNinjaShellScriptFilename().c_str());
    115       return true;
    116     }
    117     return false;
    118   }
    119 
    120   bool CheckStep1(const string& orig_args) {
    121 #define LOAD_INT(fp) ({                                                 \
    122         int v = LoadInt(fp);                                            \
    123         if (v < 0) {                                                    \
    124           fprintf(stderr, "incomplete kati_stamp, regenerating...\n");  \
    125           RETURN_TRUE;                                                  \
    126         }                                                               \
    127         v;                                                              \
    128       })
    129 
    130 #define LOAD_STRING(fp, s) ({                                           \
    131         if (!LoadString(fp, s)) {                                       \
    132           fprintf(stderr, "incomplete kati_stamp, regenerating...\n");  \
    133           RETURN_TRUE;                                                  \
    134         }                                                               \
    135       })
    136 
    137     const string& stamp_filename = GetNinjaStampFilename();
    138     FILE* fp = fopen(stamp_filename.c_str(), "rb");
    139     if (!fp) {
    140       if (g_flags.regen_debug)
    141         printf("%s: %s\n", stamp_filename.c_str(), strerror(errno));
    142       return true;
    143     }
    144     ScopedFile sfp(fp);
    145 
    146     double gen_time;
    147     size_t r = fread(&gen_time, sizeof(gen_time), 1, fp);
    148     gen_time_ = gen_time;
    149     if (r != 1) {
    150       fprintf(stderr, "incomplete kati_stamp, regenerating...\n");
    151       RETURN_TRUE;
    152     }
    153     if (g_flags.regen_debug)
    154       printf("Generated time: %f\n", gen_time);
    155 
    156     string s, s2;
    157     int num_files = LOAD_INT(fp);
    158     for (int i = 0; i < num_files; i++) {
    159       LOAD_STRING(fp, &s);
    160       double ts = GetTimestamp(s);
    161       if (gen_time < ts) {
    162         if (g_flags.regen_ignoring_kati_binary) {
    163           string kati_binary;
    164           GetExecutablePath(&kati_binary);
    165           if (s == kati_binary) {
    166             fprintf(stderr, "%s was modified, ignored.\n", s.c_str());
    167             continue;
    168           }
    169         }
    170         if (ShouldIgnoreDirty(s)) {
    171           if (g_flags.regen_debug)
    172             printf("file %s: ignored (%f)\n", s.c_str(), ts);
    173           continue;
    174         }
    175         if (g_flags.dump_kati_stamp)
    176           printf("file %s: dirty (%f)\n", s.c_str(), ts);
    177         else
    178           fprintf(stderr, "%s was modified, regenerating...\n", s.c_str());
    179         RETURN_TRUE;
    180       } else if (g_flags.dump_kati_stamp) {
    181         printf("file %s: clean (%f)\n", s.c_str(), ts);
    182       }
    183     }
    184 
    185     int num_undefineds = LOAD_INT(fp);
    186     for (int i = 0; i < num_undefineds; i++) {
    187       LOAD_STRING(fp, &s);
    188       if (getenv(s.c_str())) {
    189         if (g_flags.dump_kati_stamp) {
    190           printf("env %s: dirty (unset => %s)\n", s.c_str(), getenv(s.c_str()));
    191         } else {
    192           fprintf(stderr, "Environment variable %s was set, regenerating...\n",
    193                   s.c_str());
    194         }
    195         RETURN_TRUE;
    196       } else if (g_flags.dump_kati_stamp) {
    197         printf("env %s: clean (unset)\n", s.c_str());
    198       }
    199     }
    200 
    201     int num_envs = LOAD_INT(fp);
    202     for (int i = 0; i < num_envs; i++) {
    203       LOAD_STRING(fp, &s);
    204       StringPiece val(getenv(s.c_str()));
    205       LOAD_STRING(fp, &s2);
    206       if (val != s2) {
    207         if (g_flags.dump_kati_stamp) {
    208           printf("env %s: dirty (%s => %.*s)\n",
    209                  s.c_str(), s2.c_str(), SPF(val));
    210         } else {
    211           fprintf(stderr, "Environment variable %s was modified (%s => %.*s), "
    212                   "regenerating...\n",
    213                   s.c_str(), s2.c_str(), SPF(val));
    214         }
    215         RETURN_TRUE;
    216       } else if (g_flags.dump_kati_stamp) {
    217         printf("env %s: clean (%.*s)\n", s.c_str(), SPF(val));
    218       }
    219     }
    220 
    221     int num_globs = LOAD_INT(fp);
    222     string pat;
    223     for (int i = 0; i < num_globs; i++) {
    224       GlobResult* gr = new GlobResult;
    225       globs_.push_back(gr);
    226 
    227       LOAD_STRING(fp, &gr->pat);
    228       int num_files = LOAD_INT(fp);
    229       gr->result.resize(num_files);
    230       for (int j = 0; j < num_files; j++) {
    231         LOAD_STRING(fp, &gr->result[j]);
    232       }
    233     }
    234 
    235     int num_crs = LOAD_INT(fp);
    236     for (int i = 0; i < num_crs; i++) {
    237       ShellResult* sr = new ShellResult;
    238       commands_.push_back(sr);
    239       sr->op = static_cast<CommandOp>(LOAD_INT(fp));
    240       LOAD_STRING(fp, &sr->shell);
    241       LOAD_STRING(fp, &sr->shellflag);
    242       LOAD_STRING(fp, &sr->cmd);
    243       LOAD_STRING(fp, &sr->result);
    244 
    245       if (sr->op == CommandOp::FIND) {
    246         int num_missing_dirs = LOAD_INT(fp);
    247         for (int j = 0; j < num_missing_dirs; j++) {
    248           LOAD_STRING(fp, &s);
    249           sr->missing_dirs.push_back(s);
    250         }
    251         int num_files = LOAD_INT(fp);
    252         for (int j = 0; j < num_files; j++) {
    253           LOAD_STRING(fp, &s);
    254           sr->files.push_back(s);
    255         }
    256         int num_read_dirs = LOAD_INT(fp);
    257         for (int j = 0; j < num_read_dirs; j++) {
    258           LOAD_STRING(fp, &s);
    259           sr->read_dirs.push_back(s);
    260         }
    261       }
    262     }
    263 
    264     LoadString(fp, &s);
    265     if (orig_args != s) {
    266       fprintf(stderr, "arguments changed, regenerating...\n");
    267       RETURN_TRUE;
    268     }
    269 
    270     return needs_regen_;
    271   }
    272 
    273   bool CheckGlobResult(const GlobResult* gr, string* err) {
    274     COLLECT_STATS("glob time (regen)");
    275     vector<string>* files;
    276     Glob(gr->pat.c_str(), &files);
    277     bool needs_regen = files->size() != gr->result.size();
    278     for (size_t i = 0; i < gr->result.size(); i++) {
    279       if (!needs_regen) {
    280         if ((*files)[i] != gr->result[i]) {
    281           needs_regen = true;
    282           break;
    283         }
    284       }
    285     }
    286     if (needs_regen) {
    287       if (ShouldIgnoreDirty(gr->pat)) {
    288         if (g_flags.dump_kati_stamp) {
    289           printf("wildcard %s: ignored\n", gr->pat.c_str());
    290         }
    291         return false;
    292       }
    293       if (g_flags.dump_kati_stamp) {
    294         printf("wildcard %s: dirty\n", gr->pat.c_str());
    295       } else {
    296         *err = StringPrintf("wildcard(%s) was changed, regenerating...\n",
    297                             gr->pat.c_str());
    298       }
    299     } else if (g_flags.dump_kati_stamp) {
    300       printf("wildcard %s: clean\n", gr->pat.c_str());
    301     }
    302     return needs_regen;
    303   }
    304 
    305   bool ShouldRunCommand(const ShellResult* sr) {
    306     if (sr->op != CommandOp::FIND)
    307       return true;
    308 
    309     COLLECT_STATS("stat time (regen)");
    310     for (const string& dir : sr->missing_dirs) {
    311       if (Exists(dir))
    312         return true;
    313     }
    314     for (const string& file : sr->files) {
    315       if (!Exists(file))
    316         return true;
    317     }
    318     for (const string& dir : sr->read_dirs) {
    319       // We assume we rarely do a significant change for the top
    320       // directory which affects the results of find command.
    321       if (dir == "" || dir == "." || ShouldIgnoreDirty(dir))
    322         continue;
    323 
    324       struct stat st;
    325       if (lstat(dir.c_str(), &st) != 0) {
    326         return true;
    327       }
    328       double ts = GetTimestampFromStat(st);
    329       if (gen_time_ < ts) {
    330         return true;
    331       }
    332       if (S_ISLNK(st.st_mode)) {
    333         ts = GetTimestamp(dir);
    334         if (ts < 0 || gen_time_ < ts)
    335           return true;
    336       }
    337     }
    338     return false;
    339   }
    340 
    341   bool CheckShellResult(const ShellResult* sr, string* err) {
    342     if (sr->op == CommandOp::READ_MISSING) {
    343       if (Exists(sr->cmd)) {
    344         if (g_flags.dump_kati_stamp)
    345           printf("file %s: dirty\n", sr->cmd.c_str());
    346         else
    347           *err = StringPrintf("$(file <%s) was changed, regenerating...\n",
    348                               sr->cmd.c_str());
    349         return true;
    350       }
    351       if (g_flags.dump_kati_stamp)
    352         printf("file %s: clean\n", sr->cmd.c_str());
    353       return false;
    354     }
    355 
    356     if (sr->op == CommandOp::READ) {
    357       double ts = GetTimestamp(sr->cmd);
    358       if (gen_time_ < ts) {
    359         if (g_flags.dump_kati_stamp)
    360           printf("file %s: dirty\n", sr->cmd.c_str());
    361         else
    362           *err = StringPrintf("$(file <%s) was changed, regenerating...\n",
    363                               sr->cmd.c_str());
    364         return true;
    365       }
    366       if (g_flags.dump_kati_stamp)
    367         printf("file %s: clean\n", sr->cmd.c_str());
    368       return false;
    369     }
    370 
    371     if (sr->op == CommandOp::WRITE || sr->op == CommandOp::APPEND) {
    372       FILE* f = fopen(sr->cmd.c_str(), (sr->op == CommandOp::WRITE) ? "wb" : "ab");
    373       if (f == NULL) {
    374         PERROR("fopen");
    375       }
    376 
    377       if (fwrite(&sr->result[0], sr->result.size(), 1, f) != 1) {
    378         PERROR("fwrite");
    379       }
    380 
    381       if (fclose(f) != 0) {
    382         PERROR("fclose");
    383       }
    384 
    385       if (g_flags.dump_kati_stamp)
    386         printf("file %s: clean (write)\n", sr->cmd.c_str());
    387       return false;
    388     }
    389 
    390     if (!ShouldRunCommand(sr)) {
    391       if (g_flags.regen_debug)
    392         printf("shell %s: clean (no rerun)\n", sr->cmd.c_str());
    393       return false;
    394     }
    395 
    396     FindCommand fc;
    397     if (fc.Parse(sr->cmd) &&
    398         !fc.chdir.empty() && ShouldIgnoreDirty(fc.chdir)) {
    399       if (g_flags.dump_kati_stamp)
    400         printf("shell %s: ignored\n", sr->cmd.c_str());
    401       return false;
    402     }
    403 
    404     COLLECT_STATS_WITH_SLOW_REPORT("shell time (regen)", sr->cmd.c_str());
    405     string result;
    406     RunCommand(sr->shell, sr->shellflag, sr->cmd, RedirectStderr::DEV_NULL, &result);
    407     FormatForCommandSubstitution(&result);
    408     if (sr->result != result) {
    409       if (g_flags.dump_kati_stamp) {
    410         printf("shell %s: dirty\n", sr->cmd.c_str());
    411       } else {
    412         *err = StringPrintf("$(shell %s) was changed, regenerating...\n",
    413                             sr->cmd.c_str());
    414         //*err += StringPrintf("%s => %s\n", expected.c_str(), result.c_str());
    415       }
    416       return true;
    417     } else if (g_flags.regen_debug) {
    418       printf("shell %s: clean (rerun)\n", sr->cmd.c_str());
    419     }
    420     return false;
    421   }
    422 
    423   bool CheckStep2() {
    424     unique_ptr<ThreadPool> tp(NewThreadPool(g_flags.num_jobs));
    425 
    426     tp->Submit([this]() {
    427         string err;
    428         // TODO: Make glob cache thread safe and create a task for each glob.
    429         for (GlobResult* gr : globs_) {
    430           if (CheckGlobResult(gr, &err)) {
    431             unique_lock<mutex> lock(mu_);
    432             if (!needs_regen_) {
    433               needs_regen_ = true;
    434               msg_ = err;
    435             }
    436             break;
    437           }
    438         }
    439       });
    440 
    441     tp->Submit([this]() {
    442         for (ShellResult* sr : commands_) {
    443           string err;
    444           if (CheckShellResult(sr, &err)) {
    445             unique_lock<mutex> lock(mu_);
    446             if (!needs_regen_) {
    447               needs_regen_ = true;
    448               msg_ = err;
    449             }
    450           }
    451         }
    452       });
    453 
    454     tp->Wait();
    455     if (needs_regen_) {
    456       fprintf(stderr, "%s", msg_.c_str());
    457     }
    458     return needs_regen_;
    459   }
    460 
    461  private:
    462   double gen_time_;
    463   vector<GlobResult*> globs_;
    464   vector<ShellResult*> commands_;
    465   mutex mu_;
    466   bool needs_regen_;
    467   string msg_;
    468 };
    469 
    470 }  // namespace
    471 
    472 bool NeedsRegen(double start_time, const string& orig_args) {
    473   return StampChecker().NeedsRegen(start_time, orig_args);
    474 }
    475