Ninja
|
00001 // Copyright 2012 Google Inc. All Rights Reserved. 00002 // 00003 // Licensed under the Apache License, Version 2.0 (the "License"); 00004 // you may not use this file except in compliance with the License. 00005 // You may obtain a copy of the License at 00006 // 00007 // http://www.apache.org/licenses/LICENSE-2.0 00008 // 00009 // Unless required by applicable law or agreed to in writing, software 00010 // distributed under the License is distributed on an "AS IS" BASIS, 00011 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00012 // See the License for the specific language governing permissions and 00013 // limitations under the License. 00014 00015 #include "deps_log.h" 00016 00017 #include <assert.h> 00018 #include <stdio.h> 00019 #include <errno.h> 00020 #include <string.h> 00021 #ifndef _WIN32 00022 #include <unistd.h> 00023 #endif 00024 00025 #include "graph.h" 00026 #include "metrics.h" 00027 #include "state.h" 00028 #include "util.h" 00029 00030 // The version is stored as 4 bytes after the signature and also serves as a 00031 // byte order mark. Signature and version combined are 16 bytes long. 00032 const char kFileSignature[] = "# ninjadeps\n"; 00033 const int kCurrentVersion = 1; 00034 00035 DepsLog::~DepsLog() { 00036 Close(); 00037 } 00038 00039 bool DepsLog::OpenForWrite(const string& path, string* err) { 00040 if (needs_recompaction_) { 00041 Close(); 00042 if (!Recompact(path, err)) 00043 return false; 00044 } 00045 00046 file_ = fopen(path.c_str(), "ab"); 00047 if (!file_) { 00048 *err = strerror(errno); 00049 return false; 00050 } 00051 SetCloseOnExec(fileno(file_)); 00052 00053 // Opening a file in append mode doesn't set the file pointer to the file's 00054 // end on Windows. Do that explicitly. 00055 fseek(file_, 0, SEEK_END); 00056 00057 if (ftell(file_) == 0) { 00058 if (fwrite(kFileSignature, sizeof(kFileSignature) - 1, 1, file_) < 1) { 00059 *err = strerror(errno); 00060 return false; 00061 } 00062 if (fwrite(&kCurrentVersion, 4, 1, file_) < 1) { 00063 *err = strerror(errno); 00064 return false; 00065 } 00066 } 00067 00068 return true; 00069 } 00070 00071 bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, 00072 const vector<Node*>& nodes) { 00073 return RecordDeps(node, mtime, nodes.size(), 00074 nodes.empty() ? NULL : (Node**)&nodes.front()); 00075 } 00076 00077 bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, 00078 int node_count, Node** nodes) { 00079 // Track whether there's any new data to be recorded. 00080 bool made_change = false; 00081 00082 // Assign ids to all nodes that are missing one. 00083 if (node->id() < 0) { 00084 RecordId(node); 00085 made_change = true; 00086 } 00087 for (int i = 0; i < node_count; ++i) { 00088 if (nodes[i]->id() < 0) { 00089 RecordId(nodes[i]); 00090 made_change = true; 00091 } 00092 } 00093 00094 // See if the new data is different than the existing data, if any. 00095 if (!made_change) { 00096 Deps* deps = GetDeps(node); 00097 if (!deps || 00098 deps->mtime != mtime || 00099 deps->node_count != node_count) { 00100 made_change = true; 00101 } else { 00102 for (int i = 0; i < node_count; ++i) { 00103 if (deps->nodes[i] != nodes[i]) { 00104 made_change = true; 00105 break; 00106 } 00107 } 00108 } 00109 } 00110 00111 // Don't write anything if there's no new info. 00112 if (!made_change) 00113 return true; 00114 00115 // Update on-disk representation. 00116 uint16_t size = 4 * (1 + 1 + (uint16_t)node_count); 00117 size |= 0x8000; // Deps record: set high bit. 00118 fwrite(&size, 2, 1, file_); 00119 int id = node->id(); 00120 fwrite(&id, 4, 1, file_); 00121 int timestamp = mtime; 00122 fwrite(×tamp, 4, 1, file_); 00123 for (int i = 0; i < node_count; ++i) { 00124 id = nodes[i]->id(); 00125 fwrite(&id, 4, 1, file_); 00126 } 00127 00128 // Update in-memory representation. 00129 Deps* deps = new Deps(mtime, node_count); 00130 for (int i = 0; i < node_count; ++i) 00131 deps->nodes[i] = nodes[i]; 00132 UpdateDeps(node->id(), deps); 00133 00134 return true; 00135 } 00136 00137 void DepsLog::Close() { 00138 if (file_) 00139 fclose(file_); 00140 file_ = NULL; 00141 } 00142 00143 bool DepsLog::Load(const string& path, State* state, string* err) { 00144 METRIC_RECORD(".ninja_deps load"); 00145 char buf[32 << 10]; 00146 FILE* f = fopen(path.c_str(), "rb"); 00147 if (!f) { 00148 if (errno == ENOENT) 00149 return true; 00150 *err = strerror(errno); 00151 return false; 00152 } 00153 00154 bool valid_header = true; 00155 int version = 0; 00156 if (!fgets(buf, sizeof(buf), f) || fread(&version, 4, 1, f) < 1) 00157 valid_header = false; 00158 if (!valid_header || strcmp(buf, kFileSignature) != 0 || 00159 version != kCurrentVersion) { 00160 *err = "bad deps log signature or version; starting over"; 00161 fclose(f); 00162 unlink(path.c_str()); 00163 // Don't report this as a failure. An empty deps log will cause 00164 // us to rebuild the outputs anyway. 00165 return true; 00166 } 00167 00168 long offset; 00169 bool read_failed = false; 00170 int unique_dep_record_count = 0; 00171 int total_dep_record_count = 0; 00172 for (;;) { 00173 offset = ftell(f); 00174 00175 uint16_t size; 00176 if (fread(&size, 2, 1, f) < 1) { 00177 if (!feof(f)) 00178 read_failed = true; 00179 break; 00180 } 00181 bool is_deps = (size >> 15) != 0; 00182 size = size & 0x7FFF; 00183 00184 if (fread(buf, size, 1, f) < 1) { 00185 read_failed = true; 00186 break; 00187 } 00188 00189 if (is_deps) { 00190 assert(size % 4 == 0); 00191 int* deps_data = reinterpret_cast<int*>(buf); 00192 int out_id = deps_data[0]; 00193 int mtime = deps_data[1]; 00194 deps_data += 2; 00195 int deps_count = (size / 4) - 2; 00196 00197 Deps* deps = new Deps(mtime, deps_count); 00198 for (int i = 0; i < deps_count; ++i) { 00199 assert(deps_data[i] < (int)nodes_.size()); 00200 assert(nodes_[deps_data[i]]); 00201 deps->nodes[i] = nodes_[deps_data[i]]; 00202 } 00203 00204 total_dep_record_count++; 00205 if (!UpdateDeps(out_id, deps)) 00206 ++unique_dep_record_count; 00207 } else { 00208 StringPiece path(buf, size); 00209 Node* node = state->GetNode(path); 00210 assert(node->id() < 0); 00211 node->set_id(nodes_.size()); 00212 nodes_.push_back(node); 00213 } 00214 } 00215 00216 if (read_failed) { 00217 // An error occurred while loading; try to recover by truncating the 00218 // file to the last fully-read record. 00219 if (ferror(f)) { 00220 *err = strerror(ferror(f)); 00221 } else { 00222 *err = "premature end of file"; 00223 } 00224 fclose(f); 00225 00226 if (!Truncate(path.c_str(), offset, err)) 00227 return false; 00228 00229 // The truncate succeeded; we'll just report the load error as a 00230 // warning because the build can proceed. 00231 *err += "; recovering"; 00232 return true; 00233 } 00234 00235 fclose(f); 00236 00237 // Rebuild the log if there are too many dead records. 00238 int kMinCompactionEntryCount = 1000; 00239 int kCompactionRatio = 3; 00240 if (total_dep_record_count > kMinCompactionEntryCount && 00241 total_dep_record_count > unique_dep_record_count * kCompactionRatio) { 00242 needs_recompaction_ = true; 00243 } 00244 00245 return true; 00246 } 00247 00248 DepsLog::Deps* DepsLog::GetDeps(Node* node) { 00249 // Abort if the node has no id (never referenced in the deps) or if 00250 // there's no deps recorded for the node. 00251 if (node->id() < 0 || node->id() >= (int)deps_.size()) 00252 return NULL; 00253 return deps_[node->id()]; 00254 } 00255 00256 bool DepsLog::Recompact(const string& path, string* err) { 00257 METRIC_RECORD(".ninja_deps recompact"); 00258 printf("Recompacting deps...\n"); 00259 00260 string temp_path = path + ".recompact"; 00261 00262 // OpenForWrite() opens for append. Make sure it's not appending to a 00263 // left-over file from a previous recompaction attempt that crashed somehow. 00264 unlink(temp_path.c_str()); 00265 00266 DepsLog new_log; 00267 if (!new_log.OpenForWrite(temp_path, err)) 00268 return false; 00269 00270 // Clear all known ids so that new ones can be reassigned. The new indices 00271 // will refer to the ordering in new_log, not in the current log. 00272 for (vector<Node*>::iterator i = nodes_.begin(); i != nodes_.end(); ++i) 00273 (*i)->set_id(-1); 00274 00275 // Write out all deps again. 00276 for (int old_id = 0; old_id < (int)deps_.size(); ++old_id) { 00277 Deps* deps = deps_[old_id]; 00278 if (!deps) continue; // If nodes_[old_id] is a leaf, it has no deps. 00279 00280 if (!new_log.RecordDeps(nodes_[old_id], deps->mtime, 00281 deps->node_count, deps->nodes)) { 00282 new_log.Close(); 00283 return false; 00284 } 00285 } 00286 00287 new_log.Close(); 00288 00289 // All nodes now have ids that refer to new_log, so steal its data. 00290 deps_.swap(new_log.deps_); 00291 nodes_.swap(new_log.nodes_); 00292 00293 if (unlink(path.c_str()) < 0) { 00294 *err = strerror(errno); 00295 return false; 00296 } 00297 00298 if (rename(temp_path.c_str(), path.c_str()) < 0) { 00299 *err = strerror(errno); 00300 return false; 00301 } 00302 00303 return true; 00304 } 00305 00306 bool DepsLog::UpdateDeps(int out_id, Deps* deps) { 00307 if (out_id >= (int)deps_.size()) 00308 deps_.resize(out_id + 1); 00309 00310 bool delete_old = deps_[out_id] != NULL; 00311 if (delete_old) 00312 delete deps_[out_id]; 00313 deps_[out_id] = deps; 00314 return delete_old; 00315 } 00316 00317 bool DepsLog::RecordId(Node* node) { 00318 uint16_t size = (uint16_t)node->path().size(); 00319 fwrite(&size, 2, 1, file_); 00320 fwrite(node->path().data(), node->path().size(), 1, file_); 00321 00322 node->set_id(nodes_.size()); 00323 nodes_.push_back(node); 00324 00325 return true; 00326 }