Home | History | Annotate | Download | only in compiler 
      1 // Protocol Buffers - Google's data interchange format
      2 // Copyright 2008 Google Inc.  All rights reserved.
      3 // http://code.google.com/p/protobuf/
      4 //
      5 // Redistribution and use in source and binary forms, with or without
      6 // modification, are permitted provided that the following conditions are
      7 // met:
      8 //
      9 //     * Redistributions of source code must retain the above copyright
     10 // notice, this list of conditions and the following disclaimer.
     11 //     * Redistributions in binary form must reproduce the above
     12 // copyright notice, this list of conditions and the following disclaimer
     13 // in the documentation and/or other materials provided with the
     14 // distribution.
     15 //     * Neither the name of Google Inc. nor the names of its
     16 // contributors may be used to endorse or promote products derived from
     17 // this software without specific prior written permission.
     18 //
     19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     30 
     31 // Author: kenton (at) google.com (Kenton Varda)
     32 //  Based on original Protocol Buffers design by
     33 //  Sanjay Ghemawat, Jeff Dean, and others.
     34 //
     35 // Recursive descent FTW.
     36 
     37 #include <float.h>
     38 #include <google/protobuf/stubs/hash.h>
     39 #include <limits>
     40 
     41 
     42 #include <google/protobuf/compiler/parser.h>
     43 #include <google/protobuf/descriptor.h>
     44 #include <google/protobuf/descriptor.pb.h>
     45 #include <google/protobuf/wire_format.h>
     46 #include <google/protobuf/io/tokenizer.h>
     47 #include <google/protobuf/stubs/common.h>
     48 #include <google/protobuf/stubs/strutil.h>
     49 #include <google/protobuf/stubs/map-util.h>
     50 
     51 namespace google {
     52 namespace protobuf {
     53 namespace compiler {
     54 
     55 using internal::WireFormat;
     56 
     57 namespace {
     58 
     59 typedef hash_map<string, FieldDescriptorProto::Type> TypeNameMap;
     60 
     61 TypeNameMap MakeTypeNameTable() {
     62   TypeNameMap result;
     63 
     64   result["double"  ] = FieldDescriptorProto::TYPE_DOUBLE;
     65   result["float"   ] = FieldDescriptorProto::TYPE_FLOAT;
     66   result["uint64"  ] = FieldDescriptorProto::TYPE_UINT64;
     67   result["fixed64" ] = FieldDescriptorProto::TYPE_FIXED64;
     68   result["fixed32" ] = FieldDescriptorProto::TYPE_FIXED32;
     69   result["bool"    ] = FieldDescriptorProto::TYPE_BOOL;
     70   result["string"  ] = FieldDescriptorProto::TYPE_STRING;
     71   result["group"   ] = FieldDescriptorProto::TYPE_GROUP;
     72 
     73   result["bytes"   ] = FieldDescriptorProto::TYPE_BYTES;
     74   result["uint32"  ] = FieldDescriptorProto::TYPE_UINT32;
     75   result["sfixed32"] = FieldDescriptorProto::TYPE_SFIXED32;
     76   result["sfixed64"] = FieldDescriptorProto::TYPE_SFIXED64;
     77   result["int32"   ] = FieldDescriptorProto::TYPE_INT32;
     78   result["int64"   ] = FieldDescriptorProto::TYPE_INT64;
     79   result["sint32"  ] = FieldDescriptorProto::TYPE_SINT32;
     80   result["sint64"  ] = FieldDescriptorProto::TYPE_SINT64;
     81 
     82   return result;
     83 }
     84 
     85 const TypeNameMap kTypeNames = MakeTypeNameTable();
     86 
     87 }  // anonymous namespace
     88 
     89 // Makes code slightly more readable.  The meaning of "DO(foo)" is
     90 // "Execute foo and fail if it fails.", where failure is indicated by
     91 // returning false.
     92 #define DO(STATEMENT) if (STATEMENT) {} else return false
     93 
     94 // ===================================================================
     95 
     96 Parser::Parser()
     97   : input_(NULL),
     98     error_collector_(NULL),
     99     source_location_table_(NULL),
    100     had_errors_(false),
    101     require_syntax_identifier_(false),
    102     stop_after_syntax_identifier_(false) {
    103 }
    104 
    105 Parser::~Parser() {
    106 }
    107 
    108 // ===================================================================
    109 
    110 inline bool Parser::LookingAt(const char* text) {
    111   return input_->current().text == text;
    112 }
    113 
    114 inline bool Parser::LookingAtType(io::Tokenizer::TokenType token_type) {
    115   return input_->current().type == token_type;
    116 }
    117 
    118 inline bool Parser::AtEnd() {
    119   return LookingAtType(io::Tokenizer::TYPE_END);
    120 }
    121 
    122 bool Parser::TryConsume(const char* text) {
    123   if (LookingAt(text)) {
    124     input_->Next();
    125     return true;
    126   } else {
    127     return false;
    128   }
    129 }
    130 
    131 bool Parser::Consume(const char* text, const char* error) {
    132   if (TryConsume(text)) {
    133     return true;
    134   } else {
    135     AddError(error);
    136     return false;
    137   }
    138 }
    139 
    140 bool Parser::Consume(const char* text) {
    141   if (TryConsume(text)) {
    142     return true;
    143   } else {
    144     AddError("Expected \"" + string(text) + "\".");
    145     return false;
    146   }
    147 }
    148 
    149 bool Parser::ConsumeIdentifier(string* output, const char* error) {
    150   if (LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) {
    151     *output = input_->current().text;
    152     input_->Next();
    153     return true;
    154   } else {
    155     AddError(error);
    156     return false;
    157   }
    158 }
    159 
    160 bool Parser::ConsumeInteger(int* output, const char* error) {
    161   if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
    162     uint64 value = 0;
    163     if (!io::Tokenizer::ParseInteger(input_->current().text,
    164                                      kint32max, &value)) {
    165       AddError("Integer out of range.");
    166       // We still return true because we did, in fact, parse an integer.
    167     }
    168     *output = value;
    169     input_->Next();
    170     return true;
    171   } else {
    172     AddError(error);
    173     return false;
    174   }
    175 }
    176 
    177 bool Parser::ConsumeSignedInteger(int* output, const char* error) {
    178   bool is_negative = false;
    179   uint64 max_value = kint32max;
    180   if (TryConsume("-")) {
    181     is_negative = true;
    182     max_value += 1;
    183   }
    184   uint64 value = 0;
    185   DO(ConsumeInteger64(max_value, &value, error));
    186   if (is_negative) value *= -1;
    187   *output = value;
    188   return true;
    189 }
    190 
    191 bool Parser::ConsumeInteger64(uint64 max_value, uint64* output,
    192                               const char* error) {
    193   if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
    194     if (!io::Tokenizer::ParseInteger(input_->current().text, max_value,
    195                                      output)) {
    196       AddError("Integer out of range.");
    197       // We still return true because we did, in fact, parse an integer.
    198       *output = 0;
    199     }
    200     input_->Next();
    201     return true;
    202   } else {
    203     AddError(error);
    204     return false;
    205   }
    206 }
    207 
    208 bool Parser::ConsumeNumber(double* output, const char* error) {
    209   if (LookingAtType(io::Tokenizer::TYPE_FLOAT)) {
    210     *output = io::Tokenizer::ParseFloat(input_->current().text);
    211     input_->Next();
    212     return true;
    213   } else if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
    214     // Also accept integers.
    215     uint64 value = 0;
    216     if (!io::Tokenizer::ParseInteger(input_->current().text,
    217                                      kuint64max, &value)) {
    218       AddError("Integer out of range.");
    219       // We still return true because we did, in fact, parse a number.
    220     }
    221     *output = value;
    222     input_->Next();
    223     return true;
    224   } else if (LookingAt("inf")) {
    225     *output = numeric_limits<double>::infinity();
    226     input_->Next();
    227     return true;
    228   } else if (LookingAt("nan")) {
    229     *output = numeric_limits<double>::quiet_NaN();
    230     input_->Next();
    231     return true;
    232   } else {
    233     AddError(error);
    234     return false;
    235   }
    236 }
    237 
    238 bool Parser::ConsumeString(string* output, const char* error) {
    239   if (LookingAtType(io::Tokenizer::TYPE_STRING)) {
    240     io::Tokenizer::ParseString(input_->current().text, output);
    241     input_->Next();
    242     // Allow C++ like concatenation of adjacent string tokens.
    243     while (LookingAtType(io::Tokenizer::TYPE_STRING)) {
    244       io::Tokenizer::ParseStringAppend(input_->current().text, output);
    245       input_->Next();
    246     }
    247     return true;
    248   } else {
    249     AddError(error);
    250     return false;
    251   }
    252 }
    253 
    254 bool Parser::TryConsumeEndOfDeclaration(const char* text,
    255                                         const LocationRecorder* location) {
    256   if (LookingAt(text)) {
    257     string leading, trailing;
    258     input_->NextWithComments(&trailing, NULL, &leading);
    259 
    260     // Save the leading comments for next time, and recall the leading comments
    261     // from last time.
    262     leading.swap(upcoming_doc_comments_);
    263 
    264     if (location != NULL) {
    265       location->AttachComments(&leading, &trailing);
    266     }
    267     return true;
    268   } else {
    269     return false;
    270   }
    271 }
    272 
    273 bool Parser::ConsumeEndOfDeclaration(const char* text,
    274                                      const LocationRecorder* location) {
    275   if (TryConsumeEndOfDeclaration(text, location)) {
    276     return true;
    277   } else {
    278     AddError("Expected \"" + string(text) + "\".");
    279     return false;
    280   }
    281 }
    282 
    283 // -------------------------------------------------------------------
    284 
    285 void Parser::AddError(int line, int column, const string& error) {
    286   if (error_collector_ != NULL) {
    287     error_collector_->AddError(line, column, error);
    288   }
    289   had_errors_ = true;
    290 }
    291 
    292 void Parser::AddError(const string& error) {
    293   AddError(input_->current().line, input_->current().column, error);
    294 }
    295 
    296 // -------------------------------------------------------------------
    297 
    298 Parser::LocationRecorder::LocationRecorder(Parser* parser)
    299   : parser_(parser),
    300     location_(parser_->source_code_info_->add_location()) {
    301   location_->add_span(parser_->input_->current().line);
    302   location_->add_span(parser_->input_->current().column);
    303 }
    304 
    305 Parser::LocationRecorder::LocationRecorder(const LocationRecorder& parent) {
    306   Init(parent);
    307 }
    308 
    309 Parser::LocationRecorder::LocationRecorder(const LocationRecorder& parent,
    310                                            int path1) {
    311   Init(parent);
    312   AddPath(path1);
    313 }
    314 
    315 Parser::LocationRecorder::LocationRecorder(const LocationRecorder& parent,
    316                                            int path1, int path2) {
    317   Init(parent);
    318   AddPath(path1);
    319   AddPath(path2);
    320 }
    321 
    322 void Parser::LocationRecorder::Init(const LocationRecorder& parent) {
    323   parser_ = parent.parser_;
    324   location_ = parser_->source_code_info_->add_location();
    325   location_->mutable_path()->CopyFrom(parent.location_->path());
    326 
    327   location_->add_span(parser_->input_->current().line);
    328   location_->add_span(parser_->input_->current().column);
    329 }
    330 
    331 Parser::LocationRecorder::~LocationRecorder() {
    332   if (location_->span_size() <= 2) {
    333     EndAt(parser_->input_->previous());
    334   }
    335 }
    336 
    337 void Parser::LocationRecorder::AddPath(int path_component) {
    338   location_->add_path(path_component);
    339 }
    340 
    341 void Parser::LocationRecorder::StartAt(const io::Tokenizer::Token& token) {
    342   location_->set_span(0, token.line);
    343   location_->set_span(1, token.column);
    344 }
    345 
    346 void Parser::LocationRecorder::EndAt(const io::Tokenizer::Token& token) {
    347   if (token.line != location_->span(0)) {
    348     location_->add_span(token.line);
    349   }
    350   location_->add_span(token.end_column);
    351 }
    352 
    353 void Parser::LocationRecorder::RecordLegacyLocation(const Message* descriptor,
    354     DescriptorPool::ErrorCollector::ErrorLocation location) {
    355   if (parser_->source_location_table_ != NULL) {
    356     parser_->source_location_table_->Add(
    357         descriptor, location, location_->span(0), location_->span(1));
    358   }
    359 }
    360 
    361 void Parser::LocationRecorder::AttachComments(
    362     string* leading, string* trailing) const {
    363   GOOGLE_CHECK(!location_->has_leading_comments());
    364   GOOGLE_CHECK(!location_->has_trailing_comments());
    365 
    366   if (!leading->empty()) {
    367     location_->mutable_leading_comments()->swap(*leading);
    368   }
    369   if (!trailing->empty()) {
    370     location_->mutable_trailing_comments()->swap(*trailing);
    371   }
    372 }
    373 
    374 // -------------------------------------------------------------------
    375 
    376 void Parser::SkipStatement() {
    377   while (true) {
    378     if (AtEnd()) {
    379       return;
    380     } else if (LookingAtType(io::Tokenizer::TYPE_SYMBOL)) {
    381       if (TryConsumeEndOfDeclaration(";", NULL)) {
    382         return;
    383       } else if (TryConsume("{")) {
    384         SkipRestOfBlock();
    385         return;
    386       } else if (LookingAt("}")) {
    387         return;
    388       }
    389     }
    390     input_->Next();
    391   }
    392 }
    393 
    394 void Parser::SkipRestOfBlock() {
    395   while (true) {
    396     if (AtEnd()) {
    397       return;
    398     } else if (LookingAtType(io::Tokenizer::TYPE_SYMBOL)) {
    399       if (TryConsumeEndOfDeclaration("}", NULL)) {
    400         return;
    401       } else if (TryConsume("{")) {
    402         SkipRestOfBlock();
    403       }
    404     }
    405     input_->Next();
    406   }
    407 }
    408 
    409 // ===================================================================
    410 
    411 bool Parser::Parse(io::Tokenizer* input, FileDescriptorProto* file) {
    412   input_ = input;
    413   had_errors_ = false;
    414   syntax_identifier_.clear();
    415 
    416   // Note that |file| could be NULL at this point if
    417   // stop_after_syntax_identifier_ is true.  So, we conservatively allocate
    418   // SourceCodeInfo on the stack, then swap it into the FileDescriptorProto
    419   // later on.
    420   SourceCodeInfo source_code_info;
    421   source_code_info_ = &source_code_info;
    422 
    423   if (LookingAtType(io::Tokenizer::TYPE_START)) {
    424     // Advance to first token.
    425     input_->NextWithComments(NULL, NULL, &upcoming_doc_comments_);
    426   }
    427 
    428   {
    429     LocationRecorder root_location(this);
    430 
    431     if (require_syntax_identifier_ || LookingAt("syntax")) {
    432       if (!ParseSyntaxIdentifier()) {
    433         // Don't attempt to parse the file if we didn't recognize the syntax
    434         // identifier.
    435         return false;
    436       }
    437     } else if (!stop_after_syntax_identifier_) {
    438       syntax_identifier_ = "proto2";
    439     }
    440 
    441     if (stop_after_syntax_identifier_) return !had_errors_;
    442 
    443     // Repeatedly parse statements until we reach the end of the file.
    444     while (!AtEnd()) {
    445       if (!ParseTopLevelStatement(file, root_location)) {
    446         // This statement failed to parse.  Skip it, but keep looping to parse
    447         // other statements.
    448         SkipStatement();
    449 
    450         if (LookingAt("}")) {
    451           AddError("Unmatched \"}\".");
    452           input_->NextWithComments(NULL, NULL, &upcoming_doc_comments_);
    453         }
    454       }
    455     }
    456   }
    457 
    458   input_ = NULL;
    459   source_code_info_ = NULL;
    460   source_code_info.Swap(file->mutable_source_code_info());
    461   return !had_errors_;
    462 }
    463 
    464 bool Parser::ParseSyntaxIdentifier() {
    465   DO(Consume("syntax", "File must begin with 'syntax = \"proto2\";'."));
    466   DO(Consume("="));
    467   io::Tokenizer::Token syntax_token = input_->current();
    468   string syntax;
    469   DO(ConsumeString(&syntax, "Expected syntax identifier."));
    470   DO(ConsumeEndOfDeclaration(";", NULL));
    471 
    472   syntax_identifier_ = syntax;
    473 
    474   if (syntax != "proto2" && !stop_after_syntax_identifier_) {
    475     AddError(syntax_token.line, syntax_token.column,
    476       "Unrecognized syntax identifier \"" + syntax + "\".  This parser "
    477       "only recognizes \"proto2\".");
    478     return false;
    479   }
    480 
    481   return true;
    482 }
    483 
    484 bool Parser::ParseTopLevelStatement(FileDescriptorProto* file,
    485                                     const LocationRecorder& root_location) {
    486   if (TryConsumeEndOfDeclaration(";", NULL)) {
    487     // empty statement; ignore
    488     return true;
    489   } else if (LookingAt("message")) {
    490     LocationRecorder location(root_location,
    491       FileDescriptorProto::kMessageTypeFieldNumber, file->message_type_size());
    492     return ParseMessageDefinition(file->add_message_type(), location);
    493   } else if (LookingAt("enum")) {
    494     LocationRecorder location(root_location,
    495       FileDescriptorProto::kEnumTypeFieldNumber, file->enum_type_size());
    496     return ParseEnumDefinition(file->add_enum_type(), location);
    497   } else if (LookingAt("service")) {
    498     LocationRecorder location(root_location,
    499       FileDescriptorProto::kServiceFieldNumber, file->service_size());
    500     return ParseServiceDefinition(file->add_service(), location);
    501   } else if (LookingAt("extend")) {
    502     LocationRecorder location(root_location,
    503         FileDescriptorProto::kExtensionFieldNumber);
    504     return ParseExtend(file->mutable_extension(),
    505                        file->mutable_message_type(),
    506                        root_location,
    507                        FileDescriptorProto::kMessageTypeFieldNumber,
    508                        location);
    509   } else if (LookingAt("import")) {
    510     return ParseImport(file->mutable_dependency(),
    511                        file->mutable_public_dependency(),
    512                        file->mutable_weak_dependency(),
    513                        root_location);
    514   } else if (LookingAt("package")) {
    515     return ParsePackage(file, root_location);
    516   } else if (LookingAt("option")) {
    517     LocationRecorder location(root_location,
    518         FileDescriptorProto::kOptionsFieldNumber);
    519     return ParseOption(file->mutable_options(), location, OPTION_STATEMENT);
    520   } else {
    521     AddError("Expected top-level statement (e.g. \"message\").");
    522     return false;
    523   }
    524 }
    525 
    526 // -------------------------------------------------------------------
    527 // Messages
    528 
    529 bool Parser::ParseMessageDefinition(DescriptorProto* message,
    530                                     const LocationRecorder& message_location) {
    531   DO(Consume("message"));
    532   {
    533     LocationRecorder location(message_location,
    534                               DescriptorProto::kNameFieldNumber);
    535     location.RecordLegacyLocation(
    536         message, DescriptorPool::ErrorCollector::NAME);
    537     DO(ConsumeIdentifier(message->mutable_name(), "Expected message name."));
    538   }
    539   DO(ParseMessageBlock(message, message_location));
    540   return true;
    541 }
    542 
    543 namespace {
    544 
    545 const int kMaxExtensionRangeSentinel = -1;
    546 
    547 bool IsMessageSetWireFormatMessage(const DescriptorProto& message) {
    548   const MessageOptions& options = message.options();
    549   for (int i = 0; i < options.uninterpreted_option_size(); ++i) {
    550     const UninterpretedOption& uninterpreted = options.uninterpreted_option(i);
    551     if (uninterpreted.name_size() == 1 &&
    552         uninterpreted.name(0).name_part() == "message_set_wire_format" &&
    553         uninterpreted.identifier_value() == "true") {
    554       return true;
    555     }
    556   }
    557   return false;
    558 }
    559 
    560 // Modifies any extension ranges that specified 'max' as the end of the
    561 // extension range, and sets them to the type-specific maximum. The actual max
    562 // tag number can only be determined after all options have been parsed.
    563 void AdjustExtensionRangesWithMaxEndNumber(DescriptorProto* message) {
    564   const bool is_message_set = IsMessageSetWireFormatMessage(*message);
    565   const int max_extension_number = is_message_set ?
    566       kint32max :
    567       FieldDescriptor::kMaxNumber + 1;
    568   for (int i = 0; i < message->extension_range_size(); ++i) {
    569     if (message->extension_range(i).end() == kMaxExtensionRangeSentinel) {
    570       message->mutable_extension_range(i)->set_end(max_extension_number);
    571     }
    572   }
    573 }
    574 
    575 }  // namespace
    576 
    577 bool Parser::ParseMessageBlock(DescriptorProto* message,
    578                                const LocationRecorder& message_location) {
    579   DO(ConsumeEndOfDeclaration("{", &message_location));
    580 
    581   while (!TryConsumeEndOfDeclaration("}", NULL)) {
    582     if (AtEnd()) {
    583       AddError("Reached end of input in message definition (missing '}').");
    584       return false;
    585     }
    586 
    587     if (!ParseMessageStatement(message, message_location)) {
    588       // This statement failed to parse.  Skip it, but keep looping to parse
    589       // other statements.
    590       SkipStatement();
    591     }
    592   }
    593 
    594   if (message->extension_range_size() > 0) {
    595     AdjustExtensionRangesWithMaxEndNumber(message);
    596   }
    597   return true;
    598 }
    599 
    600 bool Parser::ParseMessageStatement(DescriptorProto* message,
    601                                    const LocationRecorder& message_location) {
    602   if (TryConsumeEndOfDeclaration(";", NULL)) {
    603     // empty statement; ignore
    604     return true;
    605   } else if (LookingAt("message")) {
    606     LocationRecorder location(message_location,
    607                               DescriptorProto::kNestedTypeFieldNumber,
    608                               message->nested_type_size());
    609     return ParseMessageDefinition(message->add_nested_type(), location);
    610   } else if (LookingAt("enum")) {
    611     LocationRecorder location(message_location,
    612                               DescriptorProto::kEnumTypeFieldNumber,
    613                               message->enum_type_size());
    614     return ParseEnumDefinition(message->add_enum_type(), location);
    615   } else if (LookingAt("extensions")) {
    616     LocationRecorder location(message_location,
    617                               DescriptorProto::kExtensionRangeFieldNumber);
    618     return ParseExtensions(message, location);
    619   } else if (LookingAt("extend")) {
    620     LocationRecorder location(message_location,
    621                               DescriptorProto::kExtensionFieldNumber);
    622     return ParseExtend(message->mutable_extension(),
    623                        message->mutable_nested_type(),
    624                        message_location,
    625                        DescriptorProto::kNestedTypeFieldNumber,
    626                        location);
    627   } else if (LookingAt("option")) {
    628     LocationRecorder location(message_location,
    629                               DescriptorProto::kOptionsFieldNumber);
    630     return ParseOption(message->mutable_options(), location, OPTION_STATEMENT);
    631   } else {
    632     LocationRecorder location(message_location,
    633                               DescriptorProto::kFieldFieldNumber,
    634                               message->field_size());
    635     return ParseMessageField(message->add_field(),
    636                              message->mutable_nested_type(),
    637                              message_location,
    638                              DescriptorProto::kNestedTypeFieldNumber,
    639                              location);
    640   }
    641 }
    642 
    643 bool Parser::ParseMessageField(FieldDescriptorProto* field,
    644                                RepeatedPtrField<DescriptorProto>* messages,
    645                                const LocationRecorder& parent_location,
    646                                int location_field_number_for_nested_type,
    647                                const LocationRecorder& field_location) {
    648   // Parse label and type.
    649   io::Tokenizer::Token label_token = input_->current();
    650   {
    651     LocationRecorder location(field_location,
    652                               FieldDescriptorProto::kLabelFieldNumber);
    653     FieldDescriptorProto::Label label;
    654     DO(ParseLabel(&label));
    655     field->set_label(label);
    656   }
    657 
    658   {
    659     LocationRecorder location(field_location);  // add path later
    660     location.RecordLegacyLocation(field, DescriptorPool::ErrorCollector::TYPE);
    661 
    662     FieldDescriptorProto::Type type = FieldDescriptorProto::TYPE_INT32;
    663     string type_name;
    664     DO(ParseType(&type, &type_name));
    665     if (type_name.empty()) {
    666       location.AddPath(FieldDescriptorProto::kTypeFieldNumber);
    667       field->set_type(type);
    668     } else {
    669       location.AddPath(FieldDescriptorProto::kTypeNameFieldNumber);
    670       field->set_type_name(type_name);
    671     }
    672   }
    673 
    674   // Parse name and '='.
    675   io::Tokenizer::Token name_token = input_->current();
    676   {
    677     LocationRecorder location(field_location,
    678                               FieldDescriptorProto::kNameFieldNumber);
    679     location.RecordLegacyLocation(field, DescriptorPool::ErrorCollector::NAME);
    680     DO(ConsumeIdentifier(field->mutable_name(), "Expected field name."));
    681   }
    682   DO(Consume("=", "Missing field number."));
    683 
    684   // Parse field number.
    685   {
    686     LocationRecorder location(field_location,
    687                               FieldDescriptorProto::kNumberFieldNumber);
    688     location.RecordLegacyLocation(
    689         field, DescriptorPool::ErrorCollector::NUMBER);
    690     int number;
    691     DO(ConsumeInteger(&number, "Expected field number."));
    692     field->set_number(number);
    693   }
    694 
    695   // Parse options.
    696   DO(ParseFieldOptions(field, field_location));
    697 
    698   // Deal with groups.
    699   if (field->has_type() && field->type() == FieldDescriptorProto::TYPE_GROUP) {
    700     // Awkward:  Since a group declares both a message type and a field, we
    701     //   have to create overlapping locations.
    702     LocationRecorder group_location(parent_location);
    703     group_location.StartAt(label_token);
    704     group_location.AddPath(location_field_number_for_nested_type);
    705     group_location.AddPath(messages->size());
    706 
    707     DescriptorProto* group = messages->Add();
    708     group->set_name(field->name());
    709 
    710     // Record name location to match the field name's location.
    711     {
    712       LocationRecorder location(group_location,
    713                                 DescriptorProto::kNameFieldNumber);
    714       location.StartAt(name_token);
    715       location.EndAt(name_token);
    716       location.RecordLegacyLocation(
    717           group, DescriptorPool::ErrorCollector::NAME);
    718     }
    719 
    720     // The field's type_name also comes from the name.  Confusing!
    721     {
    722       LocationRecorder location(field_location,
    723                                 FieldDescriptorProto::kTypeNameFieldNumber);
    724       location.StartAt(name_token);
    725       location.EndAt(name_token);
    726     }
    727 
    728     // As a hack for backwards-compatibility, we force the group name to start
    729     // with a capital letter and lower-case the field name.  New code should
    730     // not use groups; it should use nested messages.
    731     if (group->name()[0] < 'A' || 'Z' < group->name()[0]) {
    732       AddError(name_token.line, name_token.column,
    733         "Group names must start with a capital letter.");
    734     }
    735     LowerString(field->mutable_name());
    736 
    737     field->set_type_name(group->name());
    738     if (LookingAt("{")) {
    739       DO(ParseMessageBlock(group, group_location));
    740     } else {
    741       AddError("Missing group body.");
    742       return false;
    743     }
    744   } else {
    745     DO(ConsumeEndOfDeclaration(";", &field_location));
    746   }
    747 
    748   return true;
    749 }
    750 
    751 bool Parser::ParseFieldOptions(FieldDescriptorProto* field,
    752                                const LocationRecorder& field_location) {
    753   if (!LookingAt("[")) return true;
    754 
    755   LocationRecorder location(field_location,
    756                             FieldDescriptorProto::kOptionsFieldNumber);
    757 
    758   DO(Consume("["));
    759 
    760   // Parse field options.
    761   do {
    762     if (LookingAt("default")) {
    763       // We intentionally pass field_location rather than location here, since
    764       // the default value is not actually an option.
    765       DO(ParseDefaultAssignment(field, field_location));
    766     } else {
    767       DO(ParseOption(field->mutable_options(), location, OPTION_ASSIGNMENT));
    768     }
    769   } while (TryConsume(","));
    770 
    771   DO(Consume("]"));
    772   return true;
    773 }
    774 
    775 bool Parser::ParseDefaultAssignment(FieldDescriptorProto* field,
    776                                     const LocationRecorder& field_location) {
    777   if (field->has_default_value()) {
    778     AddError("Already set option \"default\".");
    779     field->clear_default_value();
    780   }
    781 
    782   DO(Consume("default"));
    783   DO(Consume("="));
    784 
    785   LocationRecorder location(field_location,
    786                             FieldDescriptorProto::kDefaultValueFieldNumber);
    787   location.RecordLegacyLocation(
    788       field, DescriptorPool::ErrorCollector::DEFAULT_VALUE);
    789   string* default_value = field->mutable_default_value();
    790 
    791   if (!field->has_type()) {
    792     // The field has a type name, but we don't know if it is a message or an
    793     // enum yet.  Assume an enum for now.
    794     DO(ConsumeIdentifier(default_value, "Expected identifier."));
    795     return true;
    796   }
    797 
    798   switch (field->type()) {
    799     case FieldDescriptorProto::TYPE_INT32:
    800     case FieldDescriptorProto::TYPE_INT64:
    801     case FieldDescriptorProto::TYPE_SINT32:
    802     case FieldDescriptorProto::TYPE_SINT64:
    803     case FieldDescriptorProto::TYPE_SFIXED32:
    804     case FieldDescriptorProto::TYPE_SFIXED64: {
    805       uint64 max_value = kint64max;
    806       if (field->type() == FieldDescriptorProto::TYPE_INT32 ||
    807           field->type() == FieldDescriptorProto::TYPE_SINT32 ||
    808           field->type() == FieldDescriptorProto::TYPE_SFIXED32) {
    809         max_value = kint32max;
    810       }
    811 
    812       // These types can be negative.
    813       if (TryConsume("-")) {
    814         default_value->append("-");
    815         // Two's complement always has one more negative value than positive.
    816         ++max_value;
    817       }
    818       // Parse the integer to verify that it is not out-of-range.
    819       uint64 value;
    820       DO(ConsumeInteger64(max_value, &value, "Expected integer."));
    821       // And stringify it again.
    822       default_value->append(SimpleItoa(value));
    823       break;
    824     }
    825 
    826     case FieldDescriptorProto::TYPE_UINT32:
    827     case FieldDescriptorProto::TYPE_UINT64:
    828     case FieldDescriptorProto::TYPE_FIXED32:
    829     case FieldDescriptorProto::TYPE_FIXED64: {
    830       uint64 max_value = kuint64max;
    831       if (field->type() == FieldDescriptorProto::TYPE_UINT32 ||
    832           field->type() == FieldDescriptorProto::TYPE_FIXED32) {
    833         max_value = kuint32max;
    834       }
    835 
    836       // Numeric, not negative.
    837       if (TryConsume("-")) {
    838         AddError("Unsigned field can't have negative default value.");
    839       }
    840       // Parse the integer to verify that it is not out-of-range.
    841       uint64 value;
    842       DO(ConsumeInteger64(max_value, &value, "Expected integer."));
    843       // And stringify it again.
    844       default_value->append(SimpleItoa(value));
    845       break;
    846     }
    847 
    848     case FieldDescriptorProto::TYPE_FLOAT:
    849     case FieldDescriptorProto::TYPE_DOUBLE:
    850       // These types can be negative.
    851       if (TryConsume("-")) {
    852         default_value->append("-");
    853       }
    854       // Parse the integer because we have to convert hex integers to decimal
    855       // floats.
    856       double value;
    857       DO(ConsumeNumber(&value, "Expected number."));
    858       // And stringify it again.
    859       default_value->append(SimpleDtoa(value));
    860       break;
    861 
    862     case FieldDescriptorProto::TYPE_BOOL:
    863       if (TryConsume("true")) {
    864         default_value->assign("true");
    865       } else if (TryConsume("false")) {
    866         default_value->assign("false");
    867       } else {
    868         AddError("Expected \"true\" or \"false\".");
    869         return false;
    870       }
    871       break;
    872 
    873     case FieldDescriptorProto::TYPE_STRING:
    874       DO(ConsumeString(default_value, "Expected string."));
    875       break;
    876 
    877     case FieldDescriptorProto::TYPE_BYTES:
    878       DO(ConsumeString(default_value, "Expected string."));
    879       *default_value = CEscape(*default_value);
    880       break;
    881 
    882     case FieldDescriptorProto::TYPE_ENUM:
    883       DO(ConsumeIdentifier(default_value, "Expected identifier."));
    884       break;
    885 
    886     case FieldDescriptorProto::TYPE_MESSAGE:
    887     case FieldDescriptorProto::TYPE_GROUP:
    888       AddError("Messages can't have default values.");
    889       return false;
    890   }
    891 
    892   return true;
    893 }
    894 
    895 bool Parser::ParseOptionNamePart(UninterpretedOption* uninterpreted_option,
    896                                  const LocationRecorder& part_location) {
    897   UninterpretedOption::NamePart* name = uninterpreted_option->add_name();
    898   string identifier;  // We parse identifiers into this string.
    899   if (LookingAt("(")) {  // This is an extension.
    900     DO(Consume("("));
    901 
    902     {
    903       LocationRecorder location(
    904           part_location, UninterpretedOption::NamePart::kNamePartFieldNumber);
    905       // An extension name consists of dot-separated identifiers, and may begin
    906       // with a dot.
    907       if (LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) {
    908         DO(ConsumeIdentifier(&identifier, "Expected identifier."));
    909         name->mutable_name_part()->append(identifier);
    910       }
    911       while (LookingAt(".")) {
    912         DO(Consume("."));
    913         name->mutable_name_part()->append(".");
    914         DO(ConsumeIdentifier(&identifier, "Expected identifier."));
    915         name->mutable_name_part()->append(identifier);
    916       }
    917     }
    918 
    919     DO(Consume(")"));
    920     name->set_is_extension(true);
    921   } else {  // This is a regular field.
    922     LocationRecorder location(
    923         part_location, UninterpretedOption::NamePart::kNamePartFieldNumber);
    924     DO(ConsumeIdentifier(&identifier, "Expected identifier."));
    925     name->mutable_name_part()->append(identifier);
    926     name->set_is_extension(false);
    927   }
    928   return true;
    929 }
    930 
    931 bool Parser::ParseUninterpretedBlock(string* value) {
    932   // Note that enclosing braces are not added to *value.
    933   // We do NOT use ConsumeEndOfStatement for this brace because it's delimiting
    934   // an expression, not a block of statements.
    935   DO(Consume("{"));
    936   int brace_depth = 1;
    937   while (!AtEnd()) {
    938     if (LookingAt("{")) {
    939       brace_depth++;
    940     } else if (LookingAt("}")) {
    941       brace_depth--;
    942       if (brace_depth == 0) {
    943         input_->Next();
    944         return true;
    945       }
    946     }
    947     // TODO(sanjay): Interpret line/column numbers to preserve formatting
    948     if (!value->empty()) value->push_back(' ');
    949     value->append(input_->current().text);
    950     input_->Next();
    951   }
    952   AddError("Unexpected end of stream while parsing aggregate value.");
    953   return false;
    954 }
    955 
    956 // We don't interpret the option here. Instead we store it in an
    957 // UninterpretedOption, to be interpreted later.
    958 bool Parser::ParseOption(Message* options,
    959                          const LocationRecorder& options_location,
    960                          OptionStyle style) {
    961   // Create an entry in the uninterpreted_option field.
    962   const FieldDescriptor* uninterpreted_option_field = options->GetDescriptor()->
    963       FindFieldByName("uninterpreted_option");
    964   GOOGLE_CHECK(uninterpreted_option_field != NULL)
    965       << "No field named \"uninterpreted_option\" in the Options proto.";
    966 
    967   const Reflection* reflection = options->GetReflection();
    968 
    969   LocationRecorder location(
    970       options_location, uninterpreted_option_field->number(),
    971       reflection->FieldSize(*options, uninterpreted_option_field));
    972 
    973   if (style == OPTION_STATEMENT) {
    974     DO(Consume("option"));
    975   }
    976 
    977   UninterpretedOption* uninterpreted_option = down_cast<UninterpretedOption*>(
    978       options->GetReflection()->AddMessage(options,
    979                                            uninterpreted_option_field));
    980 
    981   // Parse dot-separated name.
    982   {
    983     LocationRecorder name_location(location,
    984                                    UninterpretedOption::kNameFieldNumber);
    985     name_location.RecordLegacyLocation(
    986         uninterpreted_option, DescriptorPool::ErrorCollector::OPTION_NAME);
    987 
    988     {
    989       LocationRecorder part_location(name_location,
    990                                      uninterpreted_option->name_size());
    991       DO(ParseOptionNamePart(uninterpreted_option, part_location));
    992     }
    993 
    994     while (LookingAt(".")) {
    995       DO(Consume("."));
    996       LocationRecorder part_location(name_location,
    997                                      uninterpreted_option->name_size());
    998       DO(ParseOptionNamePart(uninterpreted_option, part_location));
    999     }
   1000   }
   1001 
   1002   DO(Consume("="));
   1003 
   1004   {
   1005     LocationRecorder value_location(location);
   1006     value_location.RecordLegacyLocation(
   1007         uninterpreted_option, DescriptorPool::ErrorCollector::OPTION_VALUE);
   1008 
   1009     // All values are a single token, except for negative numbers, which consist
   1010     // of a single '-' symbol, followed by a positive number.
   1011     bool is_negative = TryConsume("-");
   1012 
   1013     switch (input_->current().type) {
   1014       case io::Tokenizer::TYPE_START:
   1015         GOOGLE_LOG(FATAL) << "Trying to read value before any tokens have been read.";
   1016         return false;
   1017 
   1018       case io::Tokenizer::TYPE_END:
   1019         AddError("Unexpected end of stream while parsing option value.");
   1020         return false;
   1021 
   1022       case io::Tokenizer::TYPE_IDENTIFIER: {
   1023         value_location.AddPath(
   1024             UninterpretedOption::kIdentifierValueFieldNumber);
   1025         if (is_negative) {
   1026           AddError("Invalid '-' symbol before identifier.");
   1027           return false;
   1028         }
   1029         string value;
   1030         DO(ConsumeIdentifier(&value, "Expected identifier."));
   1031         uninterpreted_option->set_identifier_value(value);
   1032         break;
   1033       }
   1034 
   1035       case io::Tokenizer::TYPE_INTEGER: {
   1036         uint64 value;
   1037         uint64 max_value =
   1038             is_negative ? static_cast<uint64>(kint64max) + 1 : kuint64max;
   1039         DO(ConsumeInteger64(max_value, &value, "Expected integer."));
   1040         if (is_negative) {
   1041           value_location.AddPath(
   1042               UninterpretedOption::kNegativeIntValueFieldNumber);
   1043           uninterpreted_option->set_negative_int_value(
   1044               -static_cast<int64>(value));
   1045         } else {
   1046           value_location.AddPath(
   1047               UninterpretedOption::kPositiveIntValueFieldNumber);
   1048           uninterpreted_option->set_positive_int_value(value);
   1049         }
   1050         break;
   1051       }
   1052 
   1053       case io::Tokenizer::TYPE_FLOAT: {
   1054         value_location.AddPath(UninterpretedOption::kDoubleValueFieldNumber);
   1055         double value;
   1056         DO(ConsumeNumber(&value, "Expected number."));
   1057         uninterpreted_option->set_double_value(is_negative ? -value : value);
   1058         break;
   1059       }
   1060 
   1061       case io::Tokenizer::TYPE_STRING: {
   1062         value_location.AddPath(UninterpretedOption::kStringValueFieldNumber);
   1063         if (is_negative) {
   1064           AddError("Invalid '-' symbol before string.");
   1065           return false;
   1066         }
   1067         string value;
   1068         DO(ConsumeString(&value, "Expected string."));
   1069         uninterpreted_option->set_string_value(value);
   1070         break;
   1071       }
   1072 
   1073       case io::Tokenizer::TYPE_SYMBOL:
   1074         if (LookingAt("{")) {
   1075           value_location.AddPath(
   1076               UninterpretedOption::kAggregateValueFieldNumber);
   1077           DO(ParseUninterpretedBlock(
   1078               uninterpreted_option->mutable_aggregate_value()));
   1079         } else {
   1080           AddError("Expected option value.");
   1081           return false;
   1082         }
   1083         break;
   1084     }
   1085   }
   1086 
   1087   if (style == OPTION_STATEMENT) {
   1088     DO(ConsumeEndOfDeclaration(";", &location));
   1089   }
   1090 
   1091   return true;
   1092 }
   1093 
   1094 bool Parser::ParseExtensions(DescriptorProto* message,
   1095                              const LocationRecorder& extensions_location) {
   1096   // Parse the declaration.
   1097   DO(Consume("extensions"));
   1098 
   1099   do {
   1100     // Note that kExtensionRangeFieldNumber was already pushed by the parent.
   1101     LocationRecorder location(extensions_location,
   1102                               message->extension_range_size());
   1103 
   1104     DescriptorProto::ExtensionRange* range = message->add_extension_range();
   1105     location.RecordLegacyLocation(
   1106         range, DescriptorPool::ErrorCollector::NUMBER);
   1107 
   1108     int start, end;
   1109     io::Tokenizer::Token start_token;
   1110 
   1111     {
   1112       LocationRecorder start_location(
   1113           location, DescriptorProto::ExtensionRange::kStartFieldNumber);
   1114       start_token = input_->current();
   1115       DO(ConsumeInteger(&start, "Expected field number range."));
   1116     }
   1117 
   1118     if (TryConsume("to")) {
   1119       LocationRecorder end_location(
   1120           location, DescriptorProto::ExtensionRange::kEndFieldNumber);
   1121       if (TryConsume("max")) {
   1122         // Set to the sentinel value - 1 since we increment the value below.
   1123         // The actual value of the end of the range should be set with
   1124         // AdjustExtensionRangesWithMaxEndNumber.
   1125         end = kMaxExtensionRangeSentinel - 1;
   1126       } else {
   1127         DO(ConsumeInteger(&end, "Expected integer."));
   1128       }
   1129     } else {
   1130       LocationRecorder end_location(
   1131           location, DescriptorProto::ExtensionRange::kEndFieldNumber);
   1132       end_location.StartAt(start_token);
   1133       end_location.EndAt(start_token);
   1134       end = start;
   1135     }
   1136 
   1137     // Users like to specify inclusive ranges, but in code we like the end
   1138     // number to be exclusive.
   1139     ++end;
   1140 
   1141     range->set_start(start);
   1142     range->set_end(end);
   1143   } while (TryConsume(","));
   1144 
   1145   DO(ConsumeEndOfDeclaration(";", &extensions_location));
   1146   return true;
   1147 }
   1148 
   1149 bool Parser::ParseExtend(RepeatedPtrField<FieldDescriptorProto>* extensions,
   1150                          RepeatedPtrField<DescriptorProto>* messages,
   1151                          const LocationRecorder& parent_location,
   1152                          int location_field_number_for_nested_type,
   1153                          const LocationRecorder& extend_location) {
   1154   DO(Consume("extend"));
   1155 
   1156   // Parse the extendee type.
   1157   io::Tokenizer::Token extendee_start = input_->current();
   1158   string extendee;
   1159   DO(ParseUserDefinedType(&extendee));
   1160   io::Tokenizer::Token extendee_end = input_->previous();
   1161 
   1162   // Parse the block.
   1163   DO(ConsumeEndOfDeclaration("{", &extend_location));
   1164 
   1165   bool is_first = true;
   1166 
   1167   do {
   1168     if (AtEnd()) {
   1169       AddError("Reached end of input in extend definition (missing '}').");
   1170       return false;
   1171     }
   1172 
   1173     // Note that kExtensionFieldNumber was already pushed by the parent.
   1174     LocationRecorder location(extend_location, extensions->size());
   1175 
   1176     FieldDescriptorProto* field = extensions->Add();
   1177 
   1178     {
   1179       LocationRecorder extendee_location(
   1180           location, FieldDescriptorProto::kExtendeeFieldNumber);
   1181       extendee_location.StartAt(extendee_start);
   1182       extendee_location.EndAt(extendee_end);
   1183 
   1184       if (is_first) {
   1185         extendee_location.RecordLegacyLocation(
   1186             field, DescriptorPool::ErrorCollector::EXTENDEE);
   1187         is_first = false;
   1188       }
   1189     }
   1190 
   1191     field->set_extendee(extendee);
   1192 
   1193     if (!ParseMessageField(field, messages, parent_location,
   1194                            location_field_number_for_nested_type,
   1195                            location)) {
   1196       // This statement failed to parse.  Skip it, but keep looping to parse
   1197       // other statements.
   1198       SkipStatement();
   1199     }
   1200   } while (!TryConsumeEndOfDeclaration("}", NULL));
   1201 
   1202   return true;
   1203 }
   1204 
   1205 // -------------------------------------------------------------------
   1206 // Enums
   1207 
   1208 bool Parser::ParseEnumDefinition(EnumDescriptorProto* enum_type,
   1209                                  const LocationRecorder& enum_location) {
   1210   DO(Consume("enum"));
   1211 
   1212   {
   1213     LocationRecorder location(enum_location,
   1214                               EnumDescriptorProto::kNameFieldNumber);
   1215     location.RecordLegacyLocation(
   1216         enum_type, DescriptorPool::ErrorCollector::NAME);
   1217     DO(ConsumeIdentifier(enum_type->mutable_name(), "Expected enum name."));
   1218   }
   1219 
   1220   DO(ParseEnumBlock(enum_type, enum_location));
   1221   return true;
   1222 }
   1223 
   1224 bool Parser::ParseEnumBlock(EnumDescriptorProto* enum_type,
   1225                             const LocationRecorder& enum_location) {
   1226   DO(ConsumeEndOfDeclaration("{", &enum_location));
   1227 
   1228   while (!TryConsumeEndOfDeclaration("}", NULL)) {
   1229     if (AtEnd()) {
   1230       AddError("Reached end of input in enum definition (missing '}').");
   1231       return false;
   1232     }
   1233 
   1234     if (!ParseEnumStatement(enum_type, enum_location)) {
   1235       // This statement failed to parse.  Skip it, but keep looping to parse
   1236       // other statements.
   1237       SkipStatement();
   1238     }
   1239   }
   1240 
   1241   return true;
   1242 }
   1243 
   1244 bool Parser::ParseEnumStatement(EnumDescriptorProto* enum_type,
   1245                                 const LocationRecorder& enum_location) {
   1246   if (TryConsumeEndOfDeclaration(";", NULL)) {
   1247     // empty statement; ignore
   1248     return true;
   1249   } else if (LookingAt("option")) {
   1250     LocationRecorder location(enum_location,
   1251                               EnumDescriptorProto::kOptionsFieldNumber);
   1252     return ParseOption(enum_type->mutable_options(), location,
   1253                        OPTION_STATEMENT);
   1254   } else {
   1255     LocationRecorder location(enum_location,
   1256         EnumDescriptorProto::kValueFieldNumber, enum_type->value_size());
   1257     return ParseEnumConstant(enum_type->add_value(), location);
   1258   }
   1259 }
   1260 
   1261 bool Parser::ParseEnumConstant(EnumValueDescriptorProto* enum_value,
   1262                                const LocationRecorder& enum_value_location) {
   1263   // Parse name.
   1264   {
   1265     LocationRecorder location(enum_value_location,
   1266                               EnumValueDescriptorProto::kNameFieldNumber);
   1267     location.RecordLegacyLocation(
   1268         enum_value, DescriptorPool::ErrorCollector::NAME);
   1269     DO(ConsumeIdentifier(enum_value->mutable_name(),
   1270                          "Expected enum constant name."));
   1271   }
   1272 
   1273   DO(Consume("=", "Missing numeric value for enum constant."));
   1274 
   1275   // Parse value.
   1276   {
   1277     LocationRecorder location(
   1278         enum_value_location, EnumValueDescriptorProto::kNumberFieldNumber);
   1279     location.RecordLegacyLocation(
   1280         enum_value, DescriptorPool::ErrorCollector::NUMBER);
   1281 
   1282     int number;
   1283     DO(ConsumeSignedInteger(&number, "Expected integer."));
   1284     enum_value->set_number(number);
   1285   }
   1286 
   1287   DO(ParseEnumConstantOptions(enum_value, enum_value_location));
   1288 
   1289   DO(ConsumeEndOfDeclaration(";", &enum_value_location));
   1290 
   1291   return true;
   1292 }
   1293 
   1294 bool Parser::ParseEnumConstantOptions(
   1295     EnumValueDescriptorProto* value,
   1296     const LocationRecorder& enum_value_location) {
   1297   if (!LookingAt("[")) return true;
   1298 
   1299   LocationRecorder location(
   1300       enum_value_location, EnumValueDescriptorProto::kOptionsFieldNumber);
   1301 
   1302   DO(Consume("["));
   1303 
   1304   do {
   1305     DO(ParseOption(value->mutable_options(), location, OPTION_ASSIGNMENT));
   1306   } while (TryConsume(","));
   1307 
   1308   DO(Consume("]"));
   1309   return true;
   1310 }
   1311 
   1312 // -------------------------------------------------------------------
   1313 // Services
   1314 
   1315 bool Parser::ParseServiceDefinition(ServiceDescriptorProto* service,
   1316                                     const LocationRecorder& service_location) {
   1317   DO(Consume("service"));
   1318 
   1319   {
   1320     LocationRecorder location(service_location,
   1321                               ServiceDescriptorProto::kNameFieldNumber);
   1322     location.RecordLegacyLocation(
   1323         service, DescriptorPool::ErrorCollector::NAME);
   1324     DO(ConsumeIdentifier(service->mutable_name(), "Expected service name."));
   1325   }
   1326 
   1327   DO(ParseServiceBlock(service, service_location));
   1328   return true;
   1329 }
   1330 
   1331 bool Parser::ParseServiceBlock(ServiceDescriptorProto* service,
   1332                                const LocationRecorder& service_location) {
   1333   DO(ConsumeEndOfDeclaration("{", &service_location));
   1334 
   1335   while (!TryConsumeEndOfDeclaration("}", NULL)) {
   1336     if (AtEnd()) {
   1337       AddError("Reached end of input in service definition (missing '}').");
   1338       return false;
   1339     }
   1340 
   1341     if (!ParseServiceStatement(service, service_location)) {
   1342       // This statement failed to parse.  Skip it, but keep looping to parse
   1343       // other statements.
   1344       SkipStatement();
   1345     }
   1346   }
   1347 
   1348   return true;
   1349 }
   1350 
   1351 bool Parser::ParseServiceStatement(ServiceDescriptorProto* service,
   1352                                    const LocationRecorder& service_location) {
   1353   if (TryConsumeEndOfDeclaration(";", NULL)) {
   1354     // empty statement; ignore
   1355     return true;
   1356   } else if (LookingAt("option")) {
   1357     LocationRecorder location(
   1358         service_location, ServiceDescriptorProto::kOptionsFieldNumber);
   1359     return ParseOption(service->mutable_options(), location, OPTION_STATEMENT);
   1360   } else {
   1361     LocationRecorder location(service_location,
   1362         ServiceDescriptorProto::kMethodFieldNumber, service->method_size());
   1363     return ParseServiceMethod(service->add_method(), location);
   1364   }
   1365 }
   1366 
   1367 bool Parser::ParseServiceMethod(MethodDescriptorProto* method,
   1368                                 const LocationRecorder& method_location) {
   1369   DO(Consume("rpc"));
   1370 
   1371   {
   1372     LocationRecorder location(method_location,
   1373                               MethodDescriptorProto::kNameFieldNumber);
   1374     location.RecordLegacyLocation(
   1375         method, DescriptorPool::ErrorCollector::NAME);
   1376     DO(ConsumeIdentifier(method->mutable_name(), "Expected method name."));
   1377   }
   1378 
   1379   // Parse input type.
   1380   DO(Consume("("));
   1381   {
   1382     LocationRecorder location(method_location,
   1383                               MethodDescriptorProto::kInputTypeFieldNumber);
   1384     location.RecordLegacyLocation(
   1385         method, DescriptorPool::ErrorCollector::INPUT_TYPE);
   1386     DO(ParseUserDefinedType(method->mutable_input_type()));
   1387   }
   1388   DO(Consume(")"));
   1389 
   1390   // Parse output type.
   1391   DO(Consume("returns"));
   1392   DO(Consume("("));
   1393   {
   1394     LocationRecorder location(method_location,
   1395                               MethodDescriptorProto::kOutputTypeFieldNumber);
   1396     location.RecordLegacyLocation(
   1397         method, DescriptorPool::ErrorCollector::OUTPUT_TYPE);
   1398     DO(ParseUserDefinedType(method->mutable_output_type()));
   1399   }
   1400   DO(Consume(")"));
   1401 
   1402   if (LookingAt("{")) {
   1403     // Options!
   1404     DO(ParseOptions(method_location,
   1405                     MethodDescriptorProto::kOptionsFieldNumber,
   1406                     method->mutable_options()));
   1407   } else {
   1408     DO(ConsumeEndOfDeclaration(";", &method_location));
   1409   }
   1410 
   1411   return true;
   1412 }
   1413 
   1414 
   1415 bool Parser::ParseOptions(const LocationRecorder& parent_location,
   1416                           const int optionsFieldNumber,
   1417                           Message* mutable_options) {
   1418   // Options!
   1419   ConsumeEndOfDeclaration("{", &parent_location);
   1420   while (!TryConsumeEndOfDeclaration("}", NULL)) {
   1421     if (AtEnd()) {
   1422       AddError("Reached end of input in method options (missing '}').");
   1423       return false;
   1424     }
   1425 
   1426     if (TryConsumeEndOfDeclaration(";", NULL)) {
   1427       // empty statement; ignore
   1428     } else {
   1429       LocationRecorder location(parent_location,
   1430                                 optionsFieldNumber);
   1431       if (!ParseOption(mutable_options, location, OPTION_STATEMENT)) {
   1432         // This statement failed to parse.  Skip it, but keep looping to
   1433         // parse other statements.
   1434         SkipStatement();
   1435       }
   1436     }
   1437   }
   1438 
   1439   return true;
   1440 }
   1441 
   1442 // -------------------------------------------------------------------
   1443 
   1444 bool Parser::ParseLabel(FieldDescriptorProto::Label* label) {
   1445   if (TryConsume("optional")) {
   1446     *label = FieldDescriptorProto::LABEL_OPTIONAL;
   1447     return true;
   1448   } else if (TryConsume("repeated")) {
   1449     *label = FieldDescriptorProto::LABEL_REPEATED;
   1450     return true;
   1451   } else if (TryConsume("required")) {
   1452     *label = FieldDescriptorProto::LABEL_REQUIRED;
   1453     return true;
   1454   } else {
   1455     AddError("Expected \"required\", \"optional\", or \"repeated\".");
   1456     // We can actually reasonably recover here by just assuming the user
   1457     // forgot the label altogether.
   1458     *label = FieldDescriptorProto::LABEL_OPTIONAL;
   1459     return true;
   1460   }
   1461 }
   1462 
   1463 bool Parser::ParseType(FieldDescriptorProto::Type* type,
   1464                        string* type_name) {
   1465   TypeNameMap::const_iterator iter = kTypeNames.find(input_->current().text);
   1466   if (iter != kTypeNames.end()) {
   1467     *type = iter->second;
   1468     input_->Next();
   1469   } else {
   1470     DO(ParseUserDefinedType(type_name));
   1471   }
   1472   return true;
   1473 }
   1474 
   1475 bool Parser::ParseUserDefinedType(string* type_name) {
   1476   type_name->clear();
   1477 
   1478   TypeNameMap::const_iterator iter = kTypeNames.find(input_->current().text);
   1479   if (iter != kTypeNames.end()) {
   1480     // Note:  The only place enum types are allowed is for field types, but
   1481     //   if we are parsing a field type then we would not get here because
   1482     //   primitives are allowed there as well.  So this error message doesn't
   1483     //   need to account for enums.
   1484     AddError("Expected message type.");
   1485 
   1486     // Pretend to accept this type so that we can go on parsing.
   1487     *type_name = input_->current().text;
   1488     input_->Next();
   1489     return true;
   1490   }
   1491 
   1492   // A leading "." means the name is fully-qualified.
   1493   if (TryConsume(".")) type_name->append(".");
   1494 
   1495   // Consume the first part of the name.
   1496   string identifier;
   1497   DO(ConsumeIdentifier(&identifier, "Expected type name."));
   1498   type_name->append(identifier);
   1499 
   1500   // Consume more parts.
   1501   while (TryConsume(".")) {
   1502     type_name->append(".");
   1503     DO(ConsumeIdentifier(&identifier, "Expected identifier."));
   1504     type_name->append(identifier);
   1505   }
   1506 
   1507   return true;
   1508 }
   1509 
   1510 // ===================================================================
   1511 
   1512 bool Parser::ParsePackage(FileDescriptorProto* file,
   1513                           const LocationRecorder& root_location) {
   1514   if (file->has_package()) {
   1515     AddError("Multiple package definitions.");
   1516     // Don't append the new package to the old one.  Just replace it.  Not
   1517     // that it really matters since this is an error anyway.
   1518     file->clear_package();
   1519   }
   1520 
   1521   DO(Consume("package"));
   1522 
   1523   {
   1524     LocationRecorder location(root_location,
   1525                               FileDescriptorProto::kPackageFieldNumber);
   1526     location.RecordLegacyLocation(file, DescriptorPool::ErrorCollector::NAME);
   1527 
   1528     while (true) {
   1529       string identifier;
   1530       DO(ConsumeIdentifier(&identifier, "Expected identifier."));
   1531       file->mutable_package()->append(identifier);
   1532       if (!TryConsume(".")) break;
   1533       file->mutable_package()->append(".");
   1534     }
   1535 
   1536     location.EndAt(input_->previous());
   1537 
   1538     DO(ConsumeEndOfDeclaration(";", &location));
   1539   }
   1540 
   1541   return true;
   1542 }
   1543 
   1544 bool Parser::ParseImport(RepeatedPtrField<string>* dependency,
   1545                          RepeatedField<int32>* public_dependency,
   1546                          RepeatedField<int32>* weak_dependency,
   1547                          const LocationRecorder& root_location) {
   1548   DO(Consume("import"));
   1549   if (LookingAt("public")) {
   1550     LocationRecorder location(
   1551         root_location, FileDescriptorProto::kPublicDependencyFieldNumber,
   1552         public_dependency->size());
   1553     DO(Consume("public"));
   1554     *public_dependency->Add() = dependency->size();
   1555   } else if (LookingAt("weak")) {
   1556     LocationRecorder location(
   1557         root_location, FileDescriptorProto::kWeakDependencyFieldNumber,
   1558         weak_dependency->size());
   1559     DO(Consume("weak"));
   1560     *weak_dependency->Add() = dependency->size();
   1561   }
   1562   {
   1563     LocationRecorder location(root_location,
   1564                               FileDescriptorProto::kDependencyFieldNumber,
   1565                               dependency->size());
   1566     DO(ConsumeString(dependency->Add(),
   1567       "Expected a string naming the file to import."));
   1568 
   1569     location.EndAt(input_->previous());
   1570 
   1571     DO(ConsumeEndOfDeclaration(";", &location));
   1572   }
   1573   return true;
   1574 }
   1575 
   1576 // ===================================================================
   1577 
   1578 SourceLocationTable::SourceLocationTable() {}
   1579 SourceLocationTable::~SourceLocationTable() {}
   1580 
   1581 bool SourceLocationTable::Find(
   1582     const Message* descriptor,
   1583     DescriptorPool::ErrorCollector::ErrorLocation location,
   1584     int* line, int* column) const {
   1585   const pair<int, int>* result =
   1586     FindOrNull(location_map_, make_pair(descriptor, location));
   1587   if (result == NULL) {
   1588     *line   = -1;
   1589     *column = 0;
   1590     return false;
   1591   } else {
   1592     *line   = result->first;
   1593     *column = result->second;
   1594     return true;
   1595   }
   1596 }
   1597 
   1598 void SourceLocationTable::Add(
   1599     const Message* descriptor,
   1600     DescriptorPool::ErrorCollector::ErrorLocation location,
   1601     int line, int column) {
   1602   location_map_[make_pair(descriptor, location)] = make_pair(line, column);
   1603 }
   1604 
   1605 void SourceLocationTable::Clear() {
   1606   location_map_.clear();
   1607 }
   1608 
   1609 }  // namespace compiler
   1610 }  // namespace protobuf
   1611 }  // namespace google
   1612