Home | History | Annotate | Download | only in kati 
      1 // Copyright 2016 Google Inc. All rights reserved
      2 //
      3 // Licensed under the Apache License, Version 2.0 (the "License");
      4 // you may not use this file except in compliance with the License.
      5 // You may obtain a copy of the License at
      6 //
      7 //      http://www.apache.org/licenses/LICENSE-2.0
      8 //
      9 // Unless required by applicable law or agreed to in writing, software
     10 // distributed under the License is distributed on an "AS IS" BASIS,
     11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     12 // See the License for the specific language governing permissions and
     13 // limitations under the License.
     14 
     15 // +build ignore
     16 
     17 #include "regen.h"
     18 
     19 #include <sys/stat.h>
     20 
     21 #include <algorithm>
     22 #include <memory>
     23 #include <mutex>
     24 #include <vector>
     25 
     26 #include "fileutil.h"
     27 #include "find.h"
     28 #include "func.h"
     29 #include "io.h"
     30 #include "log.h"
     31 #include "ninja.h"
     32 #include "stats.h"
     33 #include "strutil.h"
     34 #include "thread_pool.h"
     35 
     36 namespace {
     37 
     38 #define RETURN_TRUE              \
     39   do {                           \
     40     if (g_flags.dump_kati_stamp) \
     41       needs_regen_ = true;       \
     42     else                         \
     43       return true;               \
     44   } while (0)
     45 
     46 bool ShouldIgnoreDirty(StringPiece s) {
     47   Pattern pat(g_flags.ignore_dirty_pattern);
     48   Pattern nopat(g_flags.no_ignore_dirty_pattern);
     49   return pat.Match(s) && !nopat.Match(s);
     50 }
     51 
     52 class StampChecker {
     53   struct GlobResult {
     54     string pat;
     55     vector<string> result;
     56   };
     57 
     58   struct ShellResult {
     59     CommandOp op;
     60     string shell;
     61     string shellflag;
     62     string cmd;
     63     string result;
     64     vector<string> missing_dirs;
     65     vector<string> files;
     66     vector<string> read_dirs;
     67   };
     68 
     69  public:
     70   StampChecker() : needs_regen_(false) {}
     71 
     72   ~StampChecker() {
     73     for (GlobResult* gr : globs_) {
     74       delete gr;
     75     }
     76     for (ShellResult* sr : commands_) {
     77       delete sr;
     78     }
     79   }
     80 
     81   bool NeedsRegen(double start_time, const string& orig_args) {
     82     if (IsMissingOutputs())
     83       RETURN_TRUE;
     84 
     85     if (CheckStep1(orig_args))
     86       RETURN_TRUE;
     87 
     88     if (CheckStep2())
     89       RETURN_TRUE;
     90 
     91     if (!needs_regen_) {
     92       FILE* fp = fopen(GetNinjaStampFilename().c_str(), "rb+");
     93       if (!fp)
     94         return true;
     95       ScopedFile sfp(fp);
     96       if (fseek(fp, 0, SEEK_SET) < 0)
     97         PERROR("fseek");
     98       size_t r = fwrite(&start_time, sizeof(start_time), 1, fp);
     99       CHECK(r == 1);
    100     }
    101     return needs_regen_;
    102   }
    103 
    104  private:
    105   bool IsMissingOutputs() {
    106     if (!Exists(GetNinjaFilename())) {
    107       fprintf(stderr, "%s is missing, regenerating...\n",
    108               GetNinjaFilename().c_str());
    109       return true;
    110     }
    111     if (!Exists(GetNinjaShellScriptFilename())) {
    112       fprintf(stderr, "%s is missing, regenerating...\n",
    113               GetNinjaShellScriptFilename().c_str());
    114       return true;
    115     }
    116     return false;
    117   }
    118 
    119   bool CheckStep1(const string& orig_args) {
    120 #define LOAD_INT(fp)                                               \
    121   ({                                                               \
    122     int v = LoadInt(fp);                                           \
    123     if (v < 0) {                                                   \
    124       fprintf(stderr, "incomplete kati_stamp, regenerating...\n"); \
    125       RETURN_TRUE;                                                 \
    126     }                                                              \
    127     v;                                                             \
    128   })
    129 
    130 #define LOAD_STRING(fp, s)                                         \
    131   ({                                                               \
    132     if (!LoadString(fp, s)) {                                      \
    133       fprintf(stderr, "incomplete kati_stamp, regenerating...\n"); \
    134       RETURN_TRUE;                                                 \
    135     }                                                              \
    136   })
    137 
    138     const string& stamp_filename = GetNinjaStampFilename();
    139     FILE* fp = fopen(stamp_filename.c_str(), "rb");
    140     if (!fp) {
    141       if (g_flags.regen_debug)
    142         printf("%s: %s\n", stamp_filename.c_str(), strerror(errno));
    143       return true;
    144     }
    145     ScopedFile sfp(fp);
    146 
    147     double gen_time;
    148     size_t r = fread(&gen_time, sizeof(gen_time), 1, fp);
    149     gen_time_ = gen_time;
    150     if (r != 1) {
    151       fprintf(stderr, "incomplete kati_stamp, regenerating...\n");
    152       RETURN_TRUE;
    153     }
    154     if (g_flags.regen_debug)
    155       printf("Generated time: %f\n", gen_time);
    156 
    157     string s, s2;
    158     int num_files = LOAD_INT(fp);
    159     for (int i = 0; i < num_files; i++) {
    160       LOAD_STRING(fp, &s);
    161       double ts = GetTimestamp(s);
    162       if (gen_time < ts) {
    163         if (g_flags.regen_ignoring_kati_binary) {
    164           string kati_binary;
    165           GetExecutablePath(&kati_binary);
    166           if (s == kati_binary) {
    167             fprintf(stderr, "%s was modified, ignored.\n", s.c_str());
    168             continue;
    169           }
    170         }
    171         if (ShouldIgnoreDirty(s)) {
    172           if (g_flags.regen_debug)
    173             printf("file %s: ignored (%f)\n", s.c_str(), ts);
    174           continue;
    175         }
    176         if (g_flags.dump_kati_stamp)
    177           printf("file %s: dirty (%f)\n", s.c_str(), ts);
    178         else
    179           fprintf(stderr, "%s was modified, regenerating...\n", s.c_str());
    180         RETURN_TRUE;
    181       } else if (g_flags.dump_kati_stamp) {
    182         printf("file %s: clean (%f)\n", s.c_str(), ts);
    183       }
    184     }
    185 
    186     int num_undefineds = LOAD_INT(fp);
    187     for (int i = 0; i < num_undefineds; i++) {
    188       LOAD_STRING(fp, &s);
    189       if (getenv(s.c_str())) {
    190         if (g_flags.dump_kati_stamp) {
    191           printf("env %s: dirty (unset => %s)\n", s.c_str(), getenv(s.c_str()));
    192         } else {
    193           fprintf(stderr, "Environment variable %s was set, regenerating...\n",
    194                   s.c_str());
    195         }
    196         RETURN_TRUE;
    197       } else if (g_flags.dump_kati_stamp) {
    198         printf("env %s: clean (unset)\n", s.c_str());
    199       }
    200     }
    201 
    202     int num_envs = LOAD_INT(fp);
    203     for (int i = 0; i < num_envs; i++) {
    204       LOAD_STRING(fp, &s);
    205       StringPiece val(getenv(s.c_str()));
    206       LOAD_STRING(fp, &s2);
    207       if (val != s2) {
    208         if (g_flags.dump_kati_stamp) {
    209           printf("env %s: dirty (%s => %.*s)\n", s.c_str(), s2.c_str(),
    210                  SPF(val));
    211         } else {
    212           fprintf(stderr,
    213                   "Environment variable %s was modified (%s => %.*s), "
    214                   "regenerating...\n",
    215                   s.c_str(), s2.c_str(), SPF(val));
    216         }
    217         RETURN_TRUE;
    218       } else if (g_flags.dump_kati_stamp) {
    219         printf("env %s: clean (%.*s)\n", s.c_str(), SPF(val));
    220       }
    221     }
    222 
    223     int num_globs = LOAD_INT(fp);
    224     string pat;
    225     for (int i = 0; i < num_globs; i++) {
    226       GlobResult* gr = new GlobResult;
    227       globs_.push_back(gr);
    228 
    229       LOAD_STRING(fp, &gr->pat);
    230       int num_files = LOAD_INT(fp);
    231       gr->result.resize(num_files);
    232       for (int j = 0; j < num_files; j++) {
    233         LOAD_STRING(fp, &gr->result[j]);
    234       }
    235     }
    236 
    237     int num_crs = LOAD_INT(fp);
    238     for (int i = 0; i < num_crs; i++) {
    239       ShellResult* sr = new ShellResult;
    240       commands_.push_back(sr);
    241       sr->op = static_cast<CommandOp>(LOAD_INT(fp));
    242       LOAD_STRING(fp, &sr->shell);
    243       LOAD_STRING(fp, &sr->shellflag);
    244       LOAD_STRING(fp, &sr->cmd);
    245       LOAD_STRING(fp, &sr->result);
    246 
    247       if (sr->op == CommandOp::FIND) {
    248         int num_missing_dirs = LOAD_INT(fp);
    249         for (int j = 0; j < num_missing_dirs; j++) {
    250           LOAD_STRING(fp, &s);
    251           sr->missing_dirs.push_back(s);
    252         }
    253         int num_files = LOAD_INT(fp);
    254         for (int j = 0; j < num_files; j++) {
    255           LOAD_STRING(fp, &s);
    256           sr->files.push_back(s);
    257         }
    258         int num_read_dirs = LOAD_INT(fp);
    259         for (int j = 0; j < num_read_dirs; j++) {
    260           LOAD_STRING(fp, &s);
    261           sr->read_dirs.push_back(s);
    262         }
    263       }
    264     }
    265 
    266     LoadString(fp, &s);
    267     if (orig_args != s) {
    268       fprintf(stderr, "arguments changed, regenerating...\n");
    269       RETURN_TRUE;
    270     }
    271 
    272     return needs_regen_;
    273   }
    274 
    275   bool CheckGlobResult(const GlobResult* gr, string* err) {
    276     COLLECT_STATS("glob time (regen)");
    277     vector<string>* files;
    278     Glob(gr->pat.c_str(), &files);
    279     bool needs_regen = files->size() != gr->result.size();
    280     for (size_t i = 0; i < gr->result.size(); i++) {
    281       if (!needs_regen) {
    282         if ((*files)[i] != gr->result[i]) {
    283           needs_regen = true;
    284           break;
    285         }
    286       }
    287     }
    288     if (needs_regen) {
    289       if (ShouldIgnoreDirty(gr->pat)) {
    290         if (g_flags.dump_kati_stamp) {
    291           printf("wildcard %s: ignored\n", gr->pat.c_str());
    292         }
    293         return false;
    294       }
    295       if (g_flags.dump_kati_stamp) {
    296         printf("wildcard %s: dirty\n", gr->pat.c_str());
    297       } else {
    298         *err = StringPrintf("wildcard(%s) was changed, regenerating...\n",
    299                             gr->pat.c_str());
    300       }
    301     } else if (g_flags.dump_kati_stamp) {
    302       printf("wildcard %s: clean\n", gr->pat.c_str());
    303     }
    304     return needs_regen;
    305   }
    306 
    307   bool ShouldRunCommand(const ShellResult* sr) {
    308     if (sr->op != CommandOp::FIND)
    309       return true;
    310 
    311     COLLECT_STATS("stat time (regen)");
    312     for (const string& dir : sr->missing_dirs) {
    313       if (Exists(dir))
    314         return true;
    315     }
    316     for (const string& file : sr->files) {
    317       if (!Exists(file))
    318         return true;
    319     }
    320     for (const string& dir : sr->read_dirs) {
    321       // We assume we rarely do a significant change for the top
    322       // directory which affects the results of find command.
    323       if (dir == "" || dir == "." || ShouldIgnoreDirty(dir))
    324         continue;
    325 
    326       struct stat st;
    327       if (lstat(dir.c_str(), &st) != 0) {
    328         return true;
    329       }
    330       double ts = GetTimestampFromStat(st);
    331       if (gen_time_ < ts) {
    332         return true;
    333       }
    334       if (S_ISLNK(st.st_mode)) {
    335         ts = GetTimestamp(dir);
    336         if (ts < 0 || gen_time_ < ts)
    337           return true;
    338       }
    339     }
    340     return false;
    341   }
    342 
    343   bool CheckShellResult(const ShellResult* sr, string* err) {
    344     if (sr->op == CommandOp::READ_MISSING) {
    345       if (Exists(sr->cmd)) {
    346         if (g_flags.dump_kati_stamp)
    347           printf("file %s: dirty\n", sr->cmd.c_str());
    348         else
    349           *err = StringPrintf("$(file <%s) was changed, regenerating...\n",
    350                               sr->cmd.c_str());
    351         return true;
    352       }
    353       if (g_flags.dump_kati_stamp)
    354         printf("file %s: clean\n", sr->cmd.c_str());
    355       return false;
    356     }
    357 
    358     if (sr->op == CommandOp::READ) {
    359       double ts = GetTimestamp(sr->cmd);
    360       if (gen_time_ < ts) {
    361         if (g_flags.dump_kati_stamp)
    362           printf("file %s: dirty\n", sr->cmd.c_str());
    363         else
    364           *err = StringPrintf("$(file <%s) was changed, regenerating...\n",
    365                               sr->cmd.c_str());
    366         return true;
    367       }
    368       if (g_flags.dump_kati_stamp)
    369         printf("file %s: clean\n", sr->cmd.c_str());
    370       return false;
    371     }
    372 
    373     if (sr->op == CommandOp::WRITE || sr->op == CommandOp::APPEND) {
    374       FILE* f =
    375           fopen(sr->cmd.c_str(), (sr->op == CommandOp::WRITE) ? "wb" : "ab");
    376       if (f == NULL) {
    377         PERROR("fopen");
    378       }
    379 
    380       if (fwrite(&sr->result[0], sr->result.size(), 1, f) != 1) {
    381         PERROR("fwrite");
    382       }
    383 
    384       if (fclose(f) != 0) {
    385         PERROR("fclose");
    386       }
    387 
    388       if (g_flags.dump_kati_stamp)
    389         printf("file %s: clean (write)\n", sr->cmd.c_str());
    390       return false;
    391     }
    392 
    393     if (!ShouldRunCommand(sr)) {
    394       if (g_flags.regen_debug)
    395         printf("shell %s: clean (no rerun)\n", sr->cmd.c_str());
    396       return false;
    397     }
    398 
    399     FindCommand fc;
    400     if (fc.Parse(sr->cmd) && !fc.chdir.empty() && ShouldIgnoreDirty(fc.chdir)) {
    401       if (g_flags.dump_kati_stamp)
    402         printf("shell %s: ignored\n", sr->cmd.c_str());
    403       return false;
    404     }
    405 
    406     COLLECT_STATS_WITH_SLOW_REPORT("shell time (regen)", sr->cmd.c_str());
    407     string result;
    408     RunCommand(sr->shell, sr->shellflag, sr->cmd, RedirectStderr::DEV_NULL,
    409                &result);
    410     FormatForCommandSubstitution(&result);
    411     if (sr->result != result) {
    412       if (g_flags.dump_kati_stamp) {
    413         printf("shell %s: dirty\n", sr->cmd.c_str());
    414       } else {
    415         *err = StringPrintf("$(shell %s) was changed, regenerating...\n",
    416                             sr->cmd.c_str());
    417         //*err += StringPrintf("%s => %s\n", expected.c_str(), result.c_str());
    418       }
    419       return true;
    420     } else if (g_flags.regen_debug) {
    421       printf("shell %s: clean (rerun)\n", sr->cmd.c_str());
    422     }
    423     return false;
    424   }
    425 
    426   bool CheckStep2() {
    427     unique_ptr<ThreadPool> tp(NewThreadPool(g_flags.num_jobs));
    428 
    429     tp->Submit([this]() {
    430       string err;
    431       // TODO: Make glob cache thread safe and create a task for each glob.
    432       for (GlobResult* gr : globs_) {
    433         if (CheckGlobResult(gr, &err)) {
    434           unique_lock<mutex> lock(mu_);
    435           if (!needs_regen_) {
    436             needs_regen_ = true;
    437             msg_ = err;
    438           }
    439           break;
    440         }
    441       }
    442     });
    443 
    444     tp->Submit([this]() {
    445       for (ShellResult* sr : commands_) {
    446         string err;
    447         if (CheckShellResult(sr, &err)) {
    448           unique_lock<mutex> lock(mu_);
    449           if (!needs_regen_) {
    450             needs_regen_ = true;
    451             msg_ = err;
    452           }
    453         }
    454       }
    455     });
    456 
    457     tp->Wait();
    458     if (needs_regen_) {
    459       fprintf(stderr, "%s", msg_.c_str());
    460     }
    461     return needs_regen_;
    462   }
    463 
    464  private:
    465   double gen_time_;
    466   vector<GlobResult*> globs_;
    467   vector<ShellResult*> commands_;
    468   mutex mu_;
    469   bool needs_regen_;
    470   string msg_;
    471 };
    472 
    473 }  // namespace
    474 
    475 bool NeedsRegen(double start_time, const string& orig_args) {
    476   return StampChecker().NeedsRegen(start_time, orig_args);
    477 }
    478