Ninja
|
00001 // Copyright 2011 Google Inc. All Rights Reserved. 00002 // 00003 // Licensed under the Apache License, Version 2.0 (the "License"); 00004 // you may not use this file except in compliance with the License. 00005 // You may obtain a copy of the License at 00006 // 00007 // http://www.apache.org/licenses/LICENSE-2.0 00008 // 00009 // Unless required by applicable law or agreed to in writing, software 00010 // distributed under the License is distributed on an "AS IS" BASIS, 00011 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00012 // See the License for the specific language governing permissions and 00013 // limitations under the License. 00014 00015 #include "build_log.h" 00016 00017 #include <errno.h> 00018 #include <stdlib.h> 00019 #include <string.h> 00020 00021 #ifndef _WIN32 00022 #define __STDC_FORMAT_MACROS 00023 #include <inttypes.h> 00024 #include <unistd.h> 00025 #endif 00026 00027 #include "build.h" 00028 #include "graph.h" 00029 #include "metrics.h" 00030 #include "util.h" 00031 00032 // Implementation details: 00033 // Each run's log appends to the log file. 00034 // To load, we run through all log entries in series, throwing away 00035 // older runs. 00036 // Once the number of redundant entries exceeds a threshold, we write 00037 // out a new file and replace the existing one with it. 00038 00039 namespace { 00040 00041 const char kFileSignature[] = "# ninja log v%d\n"; 00042 const int kOldestSupportedVersion = 4; 00043 const int kCurrentVersion = 5; 00044 00045 // 64bit MurmurHash2, by Austin Appleby 00046 #if defined(_MSC_VER) 00047 #define BIG_CONSTANT(x) (x) 00048 #else // defined(_MSC_VER) 00049 #define BIG_CONSTANT(x) (x##LLU) 00050 #endif // !defined(_MSC_VER) 00051 inline 00052 uint64_t MurmurHash64A(const void* key, size_t len) { 00053 static const uint64_t seed = 0xDECAFBADDECAFBADull; 00054 const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995); 00055 const int r = 47; 00056 uint64_t h = seed ^ (len * m); 00057 const uint64_t * data = (const uint64_t *)key; 00058 const uint64_t * end = data + (len/8); 00059 while (data != end) { 00060 uint64_t k = *data++; 00061 k *= m; 00062 k ^= k >> r; 00063 k *= m; 00064 h ^= k; 00065 h *= m; 00066 } 00067 const unsigned char* data2 = (const unsigned char*)data; 00068 switch (len & 7) 00069 { 00070 case 7: h ^= uint64_t(data2[6]) << 48; 00071 case 6: h ^= uint64_t(data2[5]) << 40; 00072 case 5: h ^= uint64_t(data2[4]) << 32; 00073 case 4: h ^= uint64_t(data2[3]) << 24; 00074 case 3: h ^= uint64_t(data2[2]) << 16; 00075 case 2: h ^= uint64_t(data2[1]) << 8; 00076 case 1: h ^= uint64_t(data2[0]); 00077 h *= m; 00078 }; 00079 h ^= h >> r; 00080 h *= m; 00081 h ^= h >> r; 00082 return h; 00083 } 00084 #undef BIG_CONSTANT 00085 00086 00087 } // namespace 00088 00089 // static 00090 uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) { 00091 return MurmurHash64A(command.str_, command.len_); 00092 } 00093 00094 BuildLog::LogEntry::LogEntry(const string& output) 00095 : output(output) {} 00096 00097 BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash, 00098 int start_time, int end_time, TimeStamp restat_mtime) 00099 : output(output), command_hash(command_hash), 00100 start_time(start_time), end_time(end_time), restat_mtime(restat_mtime) 00101 {} 00102 00103 BuildLog::BuildLog() 00104 : log_file_(NULL), needs_recompaction_(false) {} 00105 00106 BuildLog::~BuildLog() { 00107 Close(); 00108 } 00109 00110 bool BuildLog::OpenForWrite(const string& path, string* err) { 00111 if (needs_recompaction_) { 00112 Close(); 00113 if (!Recompact(path, err)) 00114 return false; 00115 } 00116 00117 log_file_ = fopen(path.c_str(), "ab"); 00118 if (!log_file_) { 00119 *err = strerror(errno); 00120 return false; 00121 } 00122 setvbuf(log_file_, NULL, _IOLBF, BUFSIZ); 00123 SetCloseOnExec(fileno(log_file_)); 00124 00125 // Opening a file in append mode doesn't set the file pointer to the file's 00126 // end on Windows. Do that explicitly. 00127 fseek(log_file_, 0, SEEK_END); 00128 00129 if (ftell(log_file_) == 0) { 00130 if (fprintf(log_file_, kFileSignature, kCurrentVersion) < 0) { 00131 *err = strerror(errno); 00132 return false; 00133 } 00134 } 00135 00136 return true; 00137 } 00138 00139 void BuildLog::RecordCommand(Edge* edge, int start_time, int end_time, 00140 TimeStamp restat_mtime) { 00141 string command = edge->EvaluateCommand(true); 00142 uint64_t command_hash = LogEntry::HashCommand(command); 00143 for (vector<Node*>::iterator out = edge->outputs_.begin(); 00144 out != edge->outputs_.end(); ++out) { 00145 const string& path = (*out)->path(); 00146 Entries::iterator i = entries_.find(path); 00147 LogEntry* log_entry; 00148 if (i != entries_.end()) { 00149 log_entry = i->second; 00150 } else { 00151 log_entry = new LogEntry(path); 00152 entries_.insert(Entries::value_type(log_entry->output, log_entry)); 00153 } 00154 log_entry->command_hash = command_hash; 00155 log_entry->start_time = start_time; 00156 log_entry->end_time = end_time; 00157 log_entry->restat_mtime = restat_mtime; 00158 00159 if (log_file_) 00160 WriteEntry(log_file_, *log_entry); 00161 } 00162 } 00163 00164 void BuildLog::Close() { 00165 if (log_file_) 00166 fclose(log_file_); 00167 log_file_ = NULL; 00168 } 00169 00170 struct LineReader { 00171 explicit LineReader(FILE* file) 00172 : file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) { 00173 memset(buf_, 0, sizeof(buf_)); 00174 } 00175 00176 // Reads a \n-terminated line from the file passed to the constructor. 00177 // On return, *line_start points to the beginning of the next line, and 00178 // *line_end points to the \n at the end of the line. If no newline is seen 00179 // in a fixed buffer size, *line_end is set to NULL. Returns false on EOF. 00180 bool ReadLine(char** line_start, char** line_end) { 00181 if (line_start_ >= buf_end_ || !line_end_) { 00182 // Buffer empty, refill. 00183 size_t size_read = fread(buf_, 1, sizeof(buf_), file_); 00184 if (!size_read) 00185 return false; 00186 line_start_ = buf_; 00187 buf_end_ = buf_ + size_read; 00188 } else { 00189 // Advance to next line in buffer. 00190 line_start_ = line_end_ + 1; 00191 } 00192 00193 line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_); 00194 if (!line_end_) { 00195 // No newline. Move rest of data to start of buffer, fill rest. 00196 size_t already_consumed = line_start_ - buf_; 00197 size_t size_rest = (buf_end_ - buf_) - already_consumed; 00198 memmove(buf_, line_start_, size_rest); 00199 00200 size_t read = fread(buf_ + size_rest, 1, sizeof(buf_) - size_rest, file_); 00201 buf_end_ = buf_ + size_rest + read; 00202 line_start_ = buf_; 00203 line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_); 00204 } 00205 00206 *line_start = line_start_; 00207 *line_end = line_end_; 00208 return true; 00209 } 00210 00211 private: 00212 FILE* file_; 00213 char buf_[256 << 10]; 00214 char* buf_end_; // Points one past the last valid byte in |buf_|. 00215 00216 char* line_start_; 00217 // Points at the next \n in buf_ after line_start, or NULL. 00218 char* line_end_; 00219 }; 00220 00221 bool BuildLog::Load(const string& path, string* err) { 00222 METRIC_RECORD(".ninja_log load"); 00223 FILE* file = fopen(path.c_str(), "r"); 00224 if (!file) { 00225 if (errno == ENOENT) 00226 return true; 00227 *err = strerror(errno); 00228 return false; 00229 } 00230 00231 int log_version = 0; 00232 int unique_entry_count = 0; 00233 int total_entry_count = 0; 00234 00235 LineReader reader(file); 00236 char* line_start = 0; 00237 char* line_end = 0; 00238 while (reader.ReadLine(&line_start, &line_end)) { 00239 if (!log_version) { 00240 sscanf(line_start, kFileSignature, &log_version); 00241 00242 if (log_version < kOldestSupportedVersion) { 00243 *err = ("build log version invalid, perhaps due to being too old; " 00244 "starting over"); 00245 fclose(file); 00246 unlink(path.c_str()); 00247 // Don't report this as a failure. An empty build log will cause 00248 // us to rebuild the outputs anyway. 00249 return true; 00250 } 00251 } 00252 00253 // If no newline was found in this chunk, read the next. 00254 if (!line_end) 00255 continue; 00256 00257 const char kFieldSeparator = '\t'; 00258 00259 char* start = line_start; 00260 char* end = (char*)memchr(start, kFieldSeparator, line_end - start); 00261 if (!end) 00262 continue; 00263 *end = 0; 00264 00265 int start_time = 0, end_time = 0; 00266 TimeStamp restat_mtime = 0; 00267 00268 start_time = atoi(start); 00269 start = end + 1; 00270 00271 end = (char*)memchr(start, kFieldSeparator, line_end - start); 00272 if (!end) 00273 continue; 00274 *end = 0; 00275 end_time = atoi(start); 00276 start = end + 1; 00277 00278 end = (char*)memchr(start, kFieldSeparator, line_end - start); 00279 if (!end) 00280 continue; 00281 *end = 0; 00282 restat_mtime = atol(start); 00283 start = end + 1; 00284 00285 end = (char*)memchr(start, kFieldSeparator, line_end - start); 00286 if (!end) 00287 continue; 00288 string output = string(start, end - start); 00289 00290 start = end + 1; 00291 end = line_end; 00292 00293 LogEntry* entry; 00294 Entries::iterator i = entries_.find(output); 00295 if (i != entries_.end()) { 00296 entry = i->second; 00297 } else { 00298 entry = new LogEntry(output); 00299 entries_.insert(Entries::value_type(entry->output, entry)); 00300 ++unique_entry_count; 00301 } 00302 ++total_entry_count; 00303 00304 entry->start_time = start_time; 00305 entry->end_time = end_time; 00306 entry->restat_mtime = restat_mtime; 00307 if (log_version >= 5) { 00308 char c = *end; *end = '\0'; 00309 entry->command_hash = (uint64_t)strtoull(start, NULL, 16); 00310 *end = c; 00311 } else { 00312 entry->command_hash = LogEntry::HashCommand(StringPiece(start, 00313 end - start)); 00314 } 00315 } 00316 fclose(file); 00317 00318 if (!line_start) { 00319 return true; // file was empty 00320 } 00321 00322 // Decide whether it's time to rebuild the log: 00323 // - if we're upgrading versions 00324 // - if it's getting large 00325 int kMinCompactionEntryCount = 100; 00326 int kCompactionRatio = 3; 00327 if (log_version < kCurrentVersion) { 00328 needs_recompaction_ = true; 00329 } else if (total_entry_count > kMinCompactionEntryCount && 00330 total_entry_count > unique_entry_count * kCompactionRatio) { 00331 needs_recompaction_ = true; 00332 } 00333 00334 return true; 00335 } 00336 00337 BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) { 00338 Entries::iterator i = entries_.find(path); 00339 if (i != entries_.end()) 00340 return i->second; 00341 return NULL; 00342 } 00343 00344 void BuildLog::WriteEntry(FILE* f, const LogEntry& entry) { 00345 fprintf(f, "%d\t%d\t%d\t%s\t%" PRIx64 "\n", 00346 entry.start_time, entry.end_time, entry.restat_mtime, 00347 entry.output.c_str(), entry.command_hash); 00348 } 00349 00350 bool BuildLog::Recompact(const string& path, string* err) { 00351 METRIC_RECORD(".ninja_log recompact"); 00352 printf("Recompacting log...\n"); 00353 00354 string temp_path = path + ".recompact"; 00355 FILE* f = fopen(temp_path.c_str(), "wb"); 00356 if (!f) { 00357 *err = strerror(errno); 00358 return false; 00359 } 00360 00361 if (fprintf(f, kFileSignature, kCurrentVersion) < 0) { 00362 *err = strerror(errno); 00363 fclose(f); 00364 return false; 00365 } 00366 00367 for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) { 00368 WriteEntry(f, *i->second); 00369 } 00370 00371 fclose(f); 00372 if (unlink(path.c_str()) < 0) { 00373 *err = strerror(errno); 00374 return false; 00375 } 00376 00377 if (rename(temp_path.c_str(), path.c_str()) < 0) { 00378 *err = strerror(errno); 00379 return false; 00380 } 00381 00382 return true; 00383 }