Ninja
build_log.cc
Go to the documentation of this file.
00001 // Copyright 2011 Google Inc. All Rights Reserved.
00002 //
00003 // Licensed under the Apache License, Version 2.0 (the "License");
00004 // you may not use this file except in compliance with the License.
00005 // You may obtain a copy of the License at
00006 //
00007 //     http://www.apache.org/licenses/LICENSE-2.0
00008 //
00009 // Unless required by applicable law or agreed to in writing, software
00010 // distributed under the License is distributed on an "AS IS" BASIS,
00011 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00012 // See the License for the specific language governing permissions and
00013 // limitations under the License.
00014 
00015 #include "build_log.h"
00016 
00017 #include <errno.h>
00018 #include <stdlib.h>
00019 #include <string.h>
00020 
00021 #ifndef _WIN32
00022 #define __STDC_FORMAT_MACROS
00023 #include <inttypes.h>
00024 #include <unistd.h>
00025 #endif
00026 
00027 #include "build.h"
00028 #include "graph.h"
00029 #include "metrics.h"
00030 #include "util.h"
00031 
00032 // Implementation details:
00033 // Each run's log appends to the log file.
00034 // To load, we run through all log entries in series, throwing away
00035 // older runs.
00036 // Once the number of redundant entries exceeds a threshold, we write
00037 // out a new file and replace the existing one with it.
00038 
00039 namespace {
00040 
00041 const char kFileSignature[] = "# ninja log v%d\n";
00042 const int kOldestSupportedVersion = 4;
00043 const int kCurrentVersion = 5;
00044 
00045 // 64bit MurmurHash2, by Austin Appleby
00046 #if defined(_MSC_VER)
00047 #define BIG_CONSTANT(x) (x)
00048 #else   // defined(_MSC_VER)
00049 #define BIG_CONSTANT(x) (x##LLU)
00050 #endif // !defined(_MSC_VER)
00051 inline
00052 uint64_t MurmurHash64A(const void* key, size_t len) {
00053   static const uint64_t seed = 0xDECAFBADDECAFBADull;
00054   const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995);
00055   const int r = 47;
00056   uint64_t h = seed ^ (len * m);
00057   const uint64_t * data = (const uint64_t *)key;
00058   const uint64_t * end = data + (len/8);
00059   while (data != end) {
00060     uint64_t k = *data++;
00061     k *= m;
00062     k ^= k >> r;
00063     k *= m;
00064     h ^= k;
00065     h *= m;
00066   }
00067   const unsigned char* data2 = (const unsigned char*)data;
00068   switch (len & 7)
00069   {
00070   case 7: h ^= uint64_t(data2[6]) << 48;
00071   case 6: h ^= uint64_t(data2[5]) << 40;
00072   case 5: h ^= uint64_t(data2[4]) << 32;
00073   case 4: h ^= uint64_t(data2[3]) << 24;
00074   case 3: h ^= uint64_t(data2[2]) << 16;
00075   case 2: h ^= uint64_t(data2[1]) << 8;
00076   case 1: h ^= uint64_t(data2[0]);
00077           h *= m;
00078   };
00079   h ^= h >> r;
00080   h *= m;
00081   h ^= h >> r;
00082   return h;
00083 }
00084 #undef BIG_CONSTANT
00085 
00086 
00087 }  // namespace
00088 
00089 // static
00090 uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) {
00091   return MurmurHash64A(command.str_, command.len_);
00092 }
00093 
00094 BuildLog::LogEntry::LogEntry(const string& output)
00095   : output(output) {}
00096 
00097 BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash,
00098   int start_time, int end_time, TimeStamp restat_mtime)
00099   : output(output), command_hash(command_hash),
00100     start_time(start_time), end_time(end_time), restat_mtime(restat_mtime)
00101 {}
00102 
00103 BuildLog::BuildLog()
00104   : log_file_(NULL), needs_recompaction_(false) {}
00105 
00106 BuildLog::~BuildLog() {
00107   Close();
00108 }
00109 
00110 bool BuildLog::OpenForWrite(const string& path, string* err) {
00111   if (needs_recompaction_) {
00112     Close();
00113     if (!Recompact(path, err))
00114       return false;
00115   }
00116 
00117   log_file_ = fopen(path.c_str(), "ab");
00118   if (!log_file_) {
00119     *err = strerror(errno);
00120     return false;
00121   }
00122   setvbuf(log_file_, NULL, _IOLBF, BUFSIZ);
00123   SetCloseOnExec(fileno(log_file_));
00124 
00125   // Opening a file in append mode doesn't set the file pointer to the file's
00126   // end on Windows. Do that explicitly.
00127   fseek(log_file_, 0, SEEK_END);
00128 
00129   if (ftell(log_file_) == 0) {
00130     if (fprintf(log_file_, kFileSignature, kCurrentVersion) < 0) {
00131       *err = strerror(errno);
00132       return false;
00133     }
00134   }
00135 
00136   return true;
00137 }
00138 
00139 void BuildLog::RecordCommand(Edge* edge, int start_time, int end_time,
00140                              TimeStamp restat_mtime) {
00141   string command = edge->EvaluateCommand(true);
00142   uint64_t command_hash = LogEntry::HashCommand(command);
00143   for (vector<Node*>::iterator out = edge->outputs_.begin();
00144        out != edge->outputs_.end(); ++out) {
00145     const string& path = (*out)->path();
00146     Entries::iterator i = entries_.find(path);
00147     LogEntry* log_entry;
00148     if (i != entries_.end()) {
00149       log_entry = i->second;
00150     } else {
00151       log_entry = new LogEntry(path);
00152       entries_.insert(Entries::value_type(log_entry->output, log_entry));
00153     }
00154     log_entry->command_hash = command_hash;
00155     log_entry->start_time = start_time;
00156     log_entry->end_time = end_time;
00157     log_entry->restat_mtime = restat_mtime;
00158 
00159     if (log_file_)
00160       WriteEntry(log_file_, *log_entry);
00161   }
00162 }
00163 
00164 void BuildLog::Close() {
00165   if (log_file_)
00166     fclose(log_file_);
00167   log_file_ = NULL;
00168 }
00169 
00170 struct LineReader {
00171   explicit LineReader(FILE* file)
00172     : file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) {
00173       memset(buf_, 0, sizeof(buf_));
00174   }
00175 
00176   // Reads a \n-terminated line from the file passed to the constructor.
00177   // On return, *line_start points to the beginning of the next line, and
00178   // *line_end points to the \n at the end of the line. If no newline is seen
00179   // in a fixed buffer size, *line_end is set to NULL. Returns false on EOF.
00180   bool ReadLine(char** line_start, char** line_end) {
00181     if (line_start_ >= buf_end_ || !line_end_) {
00182       // Buffer empty, refill.
00183       size_t size_read = fread(buf_, 1, sizeof(buf_), file_);
00184       if (!size_read)
00185         return false;
00186       line_start_ = buf_;
00187       buf_end_ = buf_ + size_read;
00188     } else {
00189       // Advance to next line in buffer.
00190       line_start_ = line_end_ + 1;
00191     }
00192 
00193     line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_);
00194     if (!line_end_) {
00195       // No newline. Move rest of data to start of buffer, fill rest.
00196       size_t already_consumed = line_start_ - buf_;
00197       size_t size_rest = (buf_end_ - buf_) - already_consumed;
00198       memmove(buf_, line_start_, size_rest);
00199 
00200       size_t read = fread(buf_ + size_rest, 1, sizeof(buf_) - size_rest, file_);
00201       buf_end_ = buf_ + size_rest + read;
00202       line_start_ = buf_;
00203       line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_);
00204     }
00205 
00206     *line_start = line_start_;
00207     *line_end = line_end_;
00208     return true;
00209   }
00210 
00211  private:
00212   FILE* file_;
00213   char buf_[256 << 10];
00214   char* buf_end_;  // Points one past the last valid byte in |buf_|.
00215 
00216   char* line_start_;
00217   // Points at the next \n in buf_ after line_start, or NULL.
00218   char* line_end_;
00219 };
00220 
00221 bool BuildLog::Load(const string& path, string* err) {
00222   METRIC_RECORD(".ninja_log load");
00223   FILE* file = fopen(path.c_str(), "r");
00224   if (!file) {
00225     if (errno == ENOENT)
00226       return true;
00227     *err = strerror(errno);
00228     return false;
00229   }
00230 
00231   int log_version = 0;
00232   int unique_entry_count = 0;
00233   int total_entry_count = 0;
00234 
00235   LineReader reader(file);
00236   char* line_start = 0;
00237   char* line_end = 0;
00238   while (reader.ReadLine(&line_start, &line_end)) {
00239     if (!log_version) {
00240       sscanf(line_start, kFileSignature, &log_version);
00241 
00242       if (log_version < kOldestSupportedVersion) {
00243         *err = ("build log version invalid, perhaps due to being too old; "
00244                 "starting over");
00245         fclose(file);
00246         unlink(path.c_str());
00247         // Don't report this as a failure.  An empty build log will cause
00248         // us to rebuild the outputs anyway.
00249         return true;
00250       }
00251     }
00252 
00253     // If no newline was found in this chunk, read the next.
00254     if (!line_end)
00255       continue;
00256 
00257     const char kFieldSeparator = '\t';
00258 
00259     char* start = line_start;
00260     char* end = (char*)memchr(start, kFieldSeparator, line_end - start);
00261     if (!end)
00262       continue;
00263     *end = 0;
00264 
00265     int start_time = 0, end_time = 0;
00266     TimeStamp restat_mtime = 0;
00267 
00268     start_time = atoi(start);
00269     start = end + 1;
00270 
00271     end = (char*)memchr(start, kFieldSeparator, line_end - start);
00272     if (!end)
00273       continue;
00274     *end = 0;
00275     end_time = atoi(start);
00276     start = end + 1;
00277 
00278     end = (char*)memchr(start, kFieldSeparator, line_end - start);
00279     if (!end)
00280       continue;
00281     *end = 0;
00282     restat_mtime = atol(start);
00283     start = end + 1;
00284 
00285     end = (char*)memchr(start, kFieldSeparator, line_end - start);
00286     if (!end)
00287       continue;
00288     string output = string(start, end - start);
00289 
00290     start = end + 1;
00291     end = line_end;
00292 
00293     LogEntry* entry;
00294     Entries::iterator i = entries_.find(output);
00295     if (i != entries_.end()) {
00296       entry = i->second;
00297     } else {
00298       entry = new LogEntry(output);
00299       entries_.insert(Entries::value_type(entry->output, entry));
00300       ++unique_entry_count;
00301     }
00302     ++total_entry_count;
00303 
00304     entry->start_time = start_time;
00305     entry->end_time = end_time;
00306     entry->restat_mtime = restat_mtime;
00307     if (log_version >= 5) {
00308       char c = *end; *end = '\0';
00309       entry->command_hash = (uint64_t)strtoull(start, NULL, 16);
00310       *end = c;
00311     } else {
00312       entry->command_hash = LogEntry::HashCommand(StringPiece(start,
00313                                                               end - start));
00314     }
00315   }
00316   fclose(file);
00317 
00318   if (!line_start) {
00319     return true; // file was empty
00320   }
00321 
00322   // Decide whether it's time to rebuild the log:
00323   // - if we're upgrading versions
00324   // - if it's getting large
00325   int kMinCompactionEntryCount = 100;
00326   int kCompactionRatio = 3;
00327   if (log_version < kCurrentVersion) {
00328     needs_recompaction_ = true;
00329   } else if (total_entry_count > kMinCompactionEntryCount &&
00330              total_entry_count > unique_entry_count * kCompactionRatio) {
00331     needs_recompaction_ = true;
00332   }
00333 
00334   return true;
00335 }
00336 
00337 BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) {
00338   Entries::iterator i = entries_.find(path);
00339   if (i != entries_.end())
00340     return i->second;
00341   return NULL;
00342 }
00343 
00344 void BuildLog::WriteEntry(FILE* f, const LogEntry& entry) {
00345   fprintf(f, "%d\t%d\t%d\t%s\t%" PRIx64 "\n",
00346           entry.start_time, entry.end_time, entry.restat_mtime,
00347           entry.output.c_str(), entry.command_hash);
00348 }
00349 
00350 bool BuildLog::Recompact(const string& path, string* err) {
00351   METRIC_RECORD(".ninja_log recompact");
00352   printf("Recompacting log...\n");
00353 
00354   string temp_path = path + ".recompact";
00355   FILE* f = fopen(temp_path.c_str(), "wb");
00356   if (!f) {
00357     *err = strerror(errno);
00358     return false;
00359   }
00360 
00361   if (fprintf(f, kFileSignature, kCurrentVersion) < 0) {
00362     *err = strerror(errno);
00363     fclose(f);
00364     return false;
00365   }
00366 
00367   for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) {
00368     WriteEntry(f, *i->second);
00369   }
00370 
00371   fclose(f);
00372   if (unlink(path.c_str()) < 0) {
00373     *err = strerror(errno);
00374     return false;
00375   }
00376 
00377   if (rename(temp_path.c_str(), path.c_str()) < 0) {
00378     *err = strerror(errno);
00379     return false;
00380   }
00381 
00382   return true;
00383 }