Ninja
manifest_parser.cc
Go to the documentation of this file.
00001 // Copyright 2011 Google Inc. All Rights Reserved.
00002 //
00003 // Licensed under the Apache License, Version 2.0 (the "License");
00004 // you may not use this file except in compliance with the License.
00005 // You may obtain a copy of the License at
00006 //
00007 //     http://www.apache.org/licenses/LICENSE-2.0
00008 //
00009 // Unless required by applicable law or agreed to in writing, software
00010 // distributed under the License is distributed on an "AS IS" BASIS,
00011 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00012 // See the License for the specific language governing permissions and
00013 // limitations under the License.
00014 
00015 #include "manifest_parser.h"
00016 
00017 #include <stdio.h>
00018 #include <vector>
00019 
00020 #include "graph.h"
00021 #include "metrics.h"
00022 #include "state.h"
00023 #include "util.h"
00024 #include "version.h"
00025 
00026 ManifestParser::ManifestParser(State* state, FileReader* file_reader)
00027   : state_(state), file_reader_(file_reader) {
00028   env_ = &state->bindings_;
00029 }
00030 bool ManifestParser::Load(const string& filename, string* err) {
00031   string contents;
00032   string read_err;
00033   if (!file_reader_->ReadFile(filename, &contents, &read_err)) {
00034     *err = "loading '" + filename + "': " + read_err;
00035     return false;
00036   }
00037   contents.resize(contents.size() + 10);
00038   return Parse(filename, contents, err);
00039 }
00040 
00041 bool ManifestParser::Parse(const string& filename, const string& input,
00042                            string* err) {
00043   METRIC_RECORD(".ninja parse");
00044   lexer_.Start(filename, input);
00045 
00046   for (;;) {
00047     Lexer::Token token = lexer_.ReadToken();
00048     switch (token) {
00049     case Lexer::POOL:
00050       if (!ParsePool(err))
00051         return false;
00052       break;
00053     case Lexer::BUILD:
00054       if (!ParseEdge(err))
00055         return false;
00056       break;
00057     case Lexer::RULE:
00058       if (!ParseRule(err))
00059         return false;
00060       break;
00061     case Lexer::DEFAULT:
00062       if (!ParseDefault(err))
00063         return false;
00064       break;
00065     case Lexer::IDENT: {
00066       lexer_.UnreadToken();
00067       string name;
00068       EvalString let_value;
00069       if (!ParseLet(&name, &let_value, err))
00070         return false;
00071       string value = let_value.Evaluate(env_);
00072       // Check ninja_required_version immediately so we can exit
00073       // before encountering any syntactic surprises.
00074       if (name == "ninja_required_version")
00075         CheckNinjaVersion(value);
00076       env_->AddBinding(name, value);
00077       break;
00078     }
00079     case Lexer::INCLUDE:
00080       if (!ParseFileInclude(false, err))
00081         return false;
00082       break;
00083     case Lexer::SUBNINJA:
00084       if (!ParseFileInclude(true, err))
00085         return false;
00086       break;
00087     case Lexer::ERROR: {
00088       return lexer_.Error(lexer_.DescribeLastError(), err);
00089     }
00090     case Lexer::TEOF:
00091       return true;
00092     case Lexer::NEWLINE:
00093       break;
00094     default:
00095       return lexer_.Error(string("unexpected ") + Lexer::TokenName(token),
00096                           err);
00097     }
00098   }
00099   return false;  // not reached
00100 }
00101 
00102 
00103 bool ManifestParser::ParsePool(string* err) {
00104   string name;
00105   if (!lexer_.ReadIdent(&name))
00106     return lexer_.Error("expected pool name", err);
00107 
00108   if (!ExpectToken(Lexer::NEWLINE, err))
00109     return false;
00110 
00111   if (state_->LookupPool(name) != NULL)
00112     return lexer_.Error("duplicate pool '" + name + "'", err);
00113 
00114   int depth = -1;
00115 
00116   while (lexer_.PeekToken(Lexer::INDENT)) {
00117     string key;
00118     EvalString value;
00119     if (!ParseLet(&key, &value, err))
00120       return false;
00121 
00122     if (key == "depth") {
00123       string depth_string = value.Evaluate(env_);
00124       depth = atol(depth_string.c_str());
00125       if (depth < 0)
00126         return lexer_.Error("invalid pool depth", err);
00127     } else {
00128       return lexer_.Error("unexpected variable '" + key + "'", err);
00129     }
00130   }
00131 
00132   if (depth < 0)
00133     return lexer_.Error("expected 'depth =' line", err);
00134 
00135   state_->AddPool(new Pool(name, depth));
00136   return true;
00137 }
00138 
00139 
00140 bool ManifestParser::ParseRule(string* err) {
00141   string name;
00142   if (!lexer_.ReadIdent(&name))
00143     return lexer_.Error("expected rule name", err);
00144 
00145   if (!ExpectToken(Lexer::NEWLINE, err))
00146     return false;
00147 
00148   if (state_->LookupRule(name) != NULL) {
00149     *err = "duplicate rule '" + name + "'";
00150     return false;
00151   }
00152 
00153   Rule* rule = new Rule(name);  // XXX scoped_ptr
00154 
00155   while (lexer_.PeekToken(Lexer::INDENT)) {
00156     string key;
00157     EvalString value;
00158     if (!ParseLet(&key, &value, err))
00159       return false;
00160 
00161     if (Rule::IsReservedBinding(key)) {
00162       rule->AddBinding(key, value);
00163     } else {
00164       // Die on other keyvals for now; revisit if we want to add a
00165       // scope here.
00166       return lexer_.Error("unexpected variable '" + key + "'", err);
00167     }
00168   }
00169 
00170   if (rule->bindings_["rspfile"].empty() !=
00171       rule->bindings_["rspfile_content"].empty()) {
00172     return lexer_.Error("rspfile and rspfile_content need to be "
00173                         "both specified", err);
00174   }
00175 
00176   if (rule->bindings_["command"].empty())
00177     return lexer_.Error("expected 'command =' line", err);
00178 
00179   state_->AddRule(rule);
00180   return true;
00181 }
00182 
00183 bool ManifestParser::ParseLet(string* key, EvalString* value, string* err) {
00184   if (!lexer_.ReadIdent(key))
00185     return false;
00186   if (!ExpectToken(Lexer::EQUALS, err))
00187     return false;
00188   if (!lexer_.ReadVarValue(value, err))
00189     return false;
00190   return true;
00191 }
00192 
00193 bool ManifestParser::ParseDefault(string* err) {
00194   EvalString eval;
00195   if (!lexer_.ReadPath(&eval, err))
00196     return false;
00197   if (eval.empty())
00198     return lexer_.Error("expected target name", err);
00199 
00200   do {
00201     string path = eval.Evaluate(env_);
00202     string path_err;
00203     if (!CanonicalizePath(&path, &path_err))
00204       return lexer_.Error(path_err, err);
00205     if (!state_->AddDefault(path, &path_err))
00206       return lexer_.Error(path_err, err);
00207 
00208     eval.Clear();
00209     if (!lexer_.ReadPath(&eval, err))
00210       return false;
00211   } while (!eval.empty());
00212 
00213   if (!ExpectToken(Lexer::NEWLINE, err))
00214     return false;
00215 
00216   return true;
00217 }
00218 
00219 bool ManifestParser::ParseEdge(string* err) {
00220   vector<EvalString> ins, outs;
00221 
00222   {
00223     EvalString out;
00224     if (!lexer_.ReadPath(&out, err))
00225       return false;
00226     if (out.empty())
00227       return lexer_.Error("expected path", err);
00228 
00229     do {
00230       outs.push_back(out);
00231 
00232       out.Clear();
00233       if (!lexer_.ReadPath(&out, err))
00234         return false;
00235     } while (!out.empty());
00236   }
00237 
00238   if (!ExpectToken(Lexer::COLON, err))
00239     return false;
00240 
00241   string rule_name;
00242   if (!lexer_.ReadIdent(&rule_name))
00243     return lexer_.Error("expected build command name", err);
00244 
00245   const Rule* rule = state_->LookupRule(rule_name);
00246   if (!rule)
00247     return lexer_.Error("unknown build rule '" + rule_name + "'", err);
00248 
00249   for (;;) {
00250     // XXX should we require one path here?
00251     EvalString in;
00252     if (!lexer_.ReadPath(&in, err))
00253       return false;
00254     if (in.empty())
00255       break;
00256     ins.push_back(in);
00257   }
00258 
00259   // Add all implicit deps, counting how many as we go.
00260   int implicit = 0;
00261   if (lexer_.PeekToken(Lexer::PIPE)) {
00262     for (;;) {
00263       EvalString in;
00264       if (!lexer_.ReadPath(&in, err))
00265         return err;
00266       if (in.empty())
00267         break;
00268       ins.push_back(in);
00269       ++implicit;
00270     }
00271   }
00272 
00273   // Add all order-only deps, counting how many as we go.
00274   int order_only = 0;
00275   if (lexer_.PeekToken(Lexer::PIPE2)) {
00276     for (;;) {
00277       EvalString in;
00278       if (!lexer_.ReadPath(&in, err))
00279         return false;
00280       if (in.empty())
00281         break;
00282       ins.push_back(in);
00283       ++order_only;
00284     }
00285   }
00286 
00287   if (!ExpectToken(Lexer::NEWLINE, err))
00288     return false;
00289 
00290   // XXX scoped_ptr to handle error case.
00291   BindingEnv* env = new BindingEnv(env_);
00292 
00293   while (lexer_.PeekToken(Lexer::INDENT)) {
00294     string key;
00295     EvalString val;
00296     if (!ParseLet(&key, &val, err))
00297       return false;
00298 
00299     env->AddBinding(key, val.Evaluate(env_));
00300   }
00301 
00302   Edge* edge = state_->AddEdge(rule);
00303   edge->env_ = env;
00304 
00305   string pool_name = edge->GetBinding("pool");
00306   if (!pool_name.empty()) {
00307     Pool* pool = state_->LookupPool(pool_name);
00308     if (pool == NULL)
00309       return lexer_.Error("unknown pool name", err);
00310     edge->pool_ = pool;
00311   }
00312 
00313   for (vector<EvalString>::iterator i = ins.begin(); i != ins.end(); ++i) {
00314     string path = i->Evaluate(env);
00315     string path_err;
00316     if (!CanonicalizePath(&path, &path_err))
00317       return lexer_.Error(path_err, err);
00318     state_->AddIn(edge, path);
00319   }
00320   for (vector<EvalString>::iterator i = outs.begin(); i != outs.end(); ++i) {
00321     string path = i->Evaluate(env);
00322     string path_err;
00323     if (!CanonicalizePath(&path, &path_err))
00324       return lexer_.Error(path_err, err);
00325     state_->AddOut(edge, path);
00326   }
00327   edge->implicit_deps_ = implicit;
00328   edge->order_only_deps_ = order_only;
00329 
00330   // Multiple outputs aren't (yet?) supported with depslog.
00331   string deps_type = edge->GetBinding("deps");
00332   if (!deps_type.empty() && edge->outputs_.size() > 1) {
00333     return lexer_.Error("multiple outputs aren't (yet?) supported by depslog; "
00334                         "bring this up on the mailing list if it affects you",
00335                         err);
00336   }
00337 
00338   return true;
00339 }
00340 
00341 bool ManifestParser::ParseFileInclude(bool new_scope, string* err) {
00342   // XXX this should use ReadPath!
00343   EvalString eval;
00344   if (!lexer_.ReadPath(&eval, err))
00345     return false;
00346   string path = eval.Evaluate(env_);
00347 
00348   string contents;
00349   string read_err;
00350   if (!file_reader_->ReadFile(path, &contents, &read_err))
00351     return lexer_.Error("loading '" + path + "': " + read_err, err);
00352 
00353   ManifestParser subparser(state_, file_reader_);
00354   if (new_scope) {
00355     subparser.env_ = new BindingEnv(env_);
00356   } else {
00357     subparser.env_ = env_;
00358   }
00359 
00360   if (!subparser.Parse(path, contents, err))
00361     return false;
00362 
00363   if (!ExpectToken(Lexer::NEWLINE, err))
00364     return false;
00365 
00366   return true;
00367 }
00368 
00369 bool ManifestParser::ExpectToken(Lexer::Token expected, string* err) {
00370   Lexer::Token token = lexer_.ReadToken();
00371   if (token != expected) {
00372     string message = string("expected ") + Lexer::TokenName(expected);
00373     message += string(", got ") + Lexer::TokenName(token);
00374     message += Lexer::TokenErrorHint(expected);
00375     return lexer_.Error(message, err);
00376   }
00377   return true;
00378 }