Ninja
deps_log.cc
Go to the documentation of this file.
00001 // Copyright 2012 Google Inc. All Rights Reserved.
00002 //
00003 // Licensed under the Apache License, Version 2.0 (the "License");
00004 // you may not use this file except in compliance with the License.
00005 // You may obtain a copy of the License at
00006 //
00007 //     http://www.apache.org/licenses/LICENSE-2.0
00008 //
00009 // Unless required by applicable law or agreed to in writing, software
00010 // distributed under the License is distributed on an "AS IS" BASIS,
00011 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00012 // See the License for the specific language governing permissions and
00013 // limitations under the License.
00014 
00015 #include "deps_log.h"
00016 
00017 #include <assert.h>
00018 #include <stdio.h>
00019 #include <errno.h>
00020 #include <string.h>
00021 #ifndef _WIN32
00022 #include <unistd.h>
00023 #endif
00024 
00025 #include "graph.h"
00026 #include "metrics.h"
00027 #include "state.h"
00028 #include "util.h"
00029 
00030 // The version is stored as 4 bytes after the signature and also serves as a
00031 // byte order mark. Signature and version combined are 16 bytes long.
00032 const char kFileSignature[] = "# ninjadeps\n";
00033 const int kCurrentVersion = 1;
00034 
00035 DepsLog::~DepsLog() {
00036   Close();
00037 }
00038 
00039 bool DepsLog::OpenForWrite(const string& path, string* err) {
00040   if (needs_recompaction_) {
00041     Close();
00042     if (!Recompact(path, err))
00043       return false;
00044   }
00045   
00046   file_ = fopen(path.c_str(), "ab");
00047   if (!file_) {
00048     *err = strerror(errno);
00049     return false;
00050   }
00051   SetCloseOnExec(fileno(file_));
00052 
00053   // Opening a file in append mode doesn't set the file pointer to the file's
00054   // end on Windows. Do that explicitly.
00055   fseek(file_, 0, SEEK_END);
00056 
00057   if (ftell(file_) == 0) {
00058     if (fwrite(kFileSignature, sizeof(kFileSignature) - 1, 1, file_) < 1) {
00059       *err = strerror(errno);
00060       return false;
00061     }
00062     if (fwrite(&kCurrentVersion, 4, 1, file_) < 1) {
00063       *err = strerror(errno);
00064       return false;
00065     }
00066   }
00067 
00068   return true;
00069 }
00070 
00071 bool DepsLog::RecordDeps(Node* node, TimeStamp mtime,
00072                          const vector<Node*>& nodes) {
00073   return RecordDeps(node, mtime, nodes.size(),
00074                     nodes.empty() ? NULL : (Node**)&nodes.front());
00075 }
00076 
00077 bool DepsLog::RecordDeps(Node* node, TimeStamp mtime,
00078                          int node_count, Node** nodes) {
00079   // Track whether there's any new data to be recorded.
00080   bool made_change = false;
00081 
00082   // Assign ids to all nodes that are missing one.
00083   if (node->id() < 0) {
00084     RecordId(node);
00085     made_change = true;
00086   }
00087   for (int i = 0; i < node_count; ++i) {
00088     if (nodes[i]->id() < 0) {
00089       RecordId(nodes[i]);
00090       made_change = true;
00091     }
00092   }
00093 
00094   // See if the new data is different than the existing data, if any.
00095   if (!made_change) {
00096     Deps* deps = GetDeps(node);
00097     if (!deps ||
00098         deps->mtime != mtime ||
00099         deps->node_count != node_count) {
00100       made_change = true;
00101     } else {
00102       for (int i = 0; i < node_count; ++i) {
00103         if (deps->nodes[i] != nodes[i]) {
00104           made_change = true;
00105           break;
00106         }
00107       }
00108     }
00109   }
00110 
00111   // Don't write anything if there's no new info.
00112   if (!made_change)
00113     return true;
00114 
00115   // Update on-disk representation.
00116   uint16_t size = 4 * (1 + 1 + (uint16_t)node_count);
00117   size |= 0x8000;  // Deps record: set high bit.
00118   fwrite(&size, 2, 1, file_);
00119   int id = node->id();
00120   fwrite(&id, 4, 1, file_);
00121   int timestamp = mtime;
00122   fwrite(&timestamp, 4, 1, file_);
00123   for (int i = 0; i < node_count; ++i) {
00124     id = nodes[i]->id();
00125     fwrite(&id, 4, 1, file_);
00126   }
00127 
00128   // Update in-memory representation.
00129   Deps* deps = new Deps(mtime, node_count);
00130   for (int i = 0; i < node_count; ++i)
00131     deps->nodes[i] = nodes[i];
00132   UpdateDeps(node->id(), deps);
00133 
00134   return true;
00135 }
00136 
00137 void DepsLog::Close() {
00138   if (file_)
00139     fclose(file_);
00140   file_ = NULL;
00141 }
00142 
00143 bool DepsLog::Load(const string& path, State* state, string* err) {
00144   METRIC_RECORD(".ninja_deps load");
00145   char buf[32 << 10];
00146   FILE* f = fopen(path.c_str(), "rb");
00147   if (!f) {
00148     if (errno == ENOENT)
00149       return true;
00150     *err = strerror(errno);
00151     return false;
00152   }
00153 
00154   bool valid_header = true;
00155   int version = 0;
00156   if (!fgets(buf, sizeof(buf), f) || fread(&version, 4, 1, f) < 1)
00157     valid_header = false;
00158   if (!valid_header || strcmp(buf, kFileSignature) != 0 ||
00159       version != kCurrentVersion) {
00160     *err = "bad deps log signature or version; starting over";
00161     fclose(f);
00162     unlink(path.c_str());
00163     // Don't report this as a failure.  An empty deps log will cause
00164     // us to rebuild the outputs anyway.
00165     return true;
00166   }
00167 
00168   long offset;
00169   bool read_failed = false;
00170   int unique_dep_record_count = 0;
00171   int total_dep_record_count = 0;
00172   for (;;) {
00173     offset = ftell(f);
00174 
00175     uint16_t size;
00176     if (fread(&size, 2, 1, f) < 1) {
00177       if (!feof(f))
00178         read_failed = true;
00179       break;
00180     }
00181     bool is_deps = (size >> 15) != 0;
00182     size = size & 0x7FFF;
00183 
00184     if (fread(buf, size, 1, f) < 1) {
00185       read_failed = true;
00186       break;
00187     }
00188 
00189     if (is_deps) {
00190       assert(size % 4 == 0);
00191       int* deps_data = reinterpret_cast<int*>(buf);
00192       int out_id = deps_data[0];
00193       int mtime = deps_data[1];
00194       deps_data += 2;
00195       int deps_count = (size / 4) - 2;
00196 
00197       Deps* deps = new Deps(mtime, deps_count);
00198       for (int i = 0; i < deps_count; ++i) {
00199         assert(deps_data[i] < (int)nodes_.size());
00200         assert(nodes_[deps_data[i]]);
00201         deps->nodes[i] = nodes_[deps_data[i]];
00202       }
00203 
00204       total_dep_record_count++;
00205       if (!UpdateDeps(out_id, deps))
00206         ++unique_dep_record_count;
00207     } else {
00208       StringPiece path(buf, size);
00209       Node* node = state->GetNode(path);
00210       assert(node->id() < 0);
00211       node->set_id(nodes_.size());
00212       nodes_.push_back(node);
00213     }
00214   }
00215 
00216   if (read_failed) {
00217     // An error occurred while loading; try to recover by truncating the
00218     // file to the last fully-read record.
00219     if (ferror(f)) {
00220       *err = strerror(ferror(f));
00221     } else {
00222       *err = "premature end of file";
00223     }
00224     fclose(f);
00225 
00226     if (!Truncate(path.c_str(), offset, err))
00227       return false;
00228 
00229     // The truncate succeeded; we'll just report the load error as a
00230     // warning because the build can proceed.
00231     *err += "; recovering";
00232     return true;
00233   }
00234 
00235   fclose(f);
00236 
00237   // Rebuild the log if there are too many dead records.
00238   int kMinCompactionEntryCount = 1000;
00239   int kCompactionRatio = 3;
00240   if (total_dep_record_count > kMinCompactionEntryCount &&
00241       total_dep_record_count > unique_dep_record_count * kCompactionRatio) {
00242     needs_recompaction_ = true;
00243   }
00244 
00245   return true;
00246 }
00247 
00248 DepsLog::Deps* DepsLog::GetDeps(Node* node) {
00249   // Abort if the node has no id (never referenced in the deps) or if
00250   // there's no deps recorded for the node.
00251   if (node->id() < 0 || node->id() >= (int)deps_.size())
00252     return NULL;
00253   return deps_[node->id()];
00254 }
00255 
00256 bool DepsLog::Recompact(const string& path, string* err) {
00257   METRIC_RECORD(".ninja_deps recompact");
00258   printf("Recompacting deps...\n");
00259 
00260   string temp_path = path + ".recompact";
00261 
00262   // OpenForWrite() opens for append.  Make sure it's not appending to a
00263   // left-over file from a previous recompaction attempt that crashed somehow.
00264   unlink(temp_path.c_str());
00265 
00266   DepsLog new_log;
00267   if (!new_log.OpenForWrite(temp_path, err))
00268     return false;
00269 
00270   // Clear all known ids so that new ones can be reassigned.  The new indices
00271   // will refer to the ordering in new_log, not in the current log.
00272   for (vector<Node*>::iterator i = nodes_.begin(); i != nodes_.end(); ++i)
00273     (*i)->set_id(-1);
00274   
00275   // Write out all deps again.
00276   for (int old_id = 0; old_id < (int)deps_.size(); ++old_id) {
00277     Deps* deps = deps_[old_id];
00278     if (!deps) continue;  // If nodes_[old_id] is a leaf, it has no deps.
00279 
00280     if (!new_log.RecordDeps(nodes_[old_id], deps->mtime,
00281                             deps->node_count, deps->nodes)) {
00282       new_log.Close();
00283       return false;
00284     }
00285   }
00286 
00287   new_log.Close();
00288 
00289   // All nodes now have ids that refer to new_log, so steal its data.
00290   deps_.swap(new_log.deps_);
00291   nodes_.swap(new_log.nodes_);
00292 
00293   if (unlink(path.c_str()) < 0) {
00294     *err = strerror(errno);
00295     return false;
00296   }
00297 
00298   if (rename(temp_path.c_str(), path.c_str()) < 0) {
00299     *err = strerror(errno);
00300     return false;
00301   }
00302 
00303   return true;
00304 }
00305 
00306 bool DepsLog::UpdateDeps(int out_id, Deps* deps) {
00307   if (out_id >= (int)deps_.size())
00308     deps_.resize(out_id + 1);
00309 
00310   bool delete_old = deps_[out_id] != NULL;
00311   if (delete_old)
00312     delete deps_[out_id];
00313   deps_[out_id] = deps;
00314   return delete_old;
00315 }
00316 
00317 bool DepsLog::RecordId(Node* node) {
00318   uint16_t size = (uint16_t)node->path().size();
00319   fwrite(&size, 2, 1, file_);
00320   fwrite(node->path().data(), node->path().size(), 1, file_);
00321 
00322   node->set_id(nodes_.size());
00323   nodes_.push_back(node);
00324 
00325   return true;
00326 }