1 /* 2 * Copyright 2014 Google Inc. All rights reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include <algorithm> 18 #include <list> 19 #include <string> 20 21 #include <math.h> 22 23 #include "flatbuffers/idl.h" 24 #include "flatbuffers/util.h" 25 26 namespace flatbuffers { 27 28 const double kPi = 3.14159265358979323846; 29 30 const char *const kTypeNames[] = { 31 // clang-format off 32 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ 33 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ 34 IDLTYPE, 35 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) 36 #undef FLATBUFFERS_TD 37 // clang-format on 38 nullptr 39 }; 40 41 const char kTypeSizes[] = { 42 // clang-format off 43 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ 44 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ 45 sizeof(CTYPE), 46 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) 47 #undef FLATBUFFERS_TD 48 // clang-format on 49 }; 50 51 // The enums in the reflection schema should match the ones we use internally. 52 // Compare the last element to check if these go out of sync. 53 static_assert(BASE_TYPE_UNION == static_cast<BaseType>(reflection::Union), 54 "enums don't match"); 55 56 // Any parsing calls have to be wrapped in this macro, which automates 57 // handling of recursive error checking a bit. It will check the received 58 // CheckedError object, and return straight away on error. 59 #define ECHECK(call) \ 60 { \ 61 auto ce = (call); \ 62 if (ce.Check()) return ce; \ 63 } 64 65 // These two functions are called hundreds of times below, so define a short 66 // form: 67 #define NEXT() ECHECK(Next()) 68 #define EXPECT(tok) ECHECK(Expect(tok)) 69 70 static bool ValidateUTF8(const std::string &str) { 71 const char *s = &str[0]; 72 const char *const sEnd = s + str.length(); 73 while (s < sEnd) { 74 if (FromUTF8(&s) < 0) { return false; } 75 } 76 return true; 77 } 78 79 // Convert an underscore_based_indentifier in to camelCase. 80 // Also uppercases the first character if first is true. 81 std::string MakeCamel(const std::string &in, bool first) { 82 std::string s; 83 for (size_t i = 0; i < in.length(); i++) { 84 if (!i && first) 85 s += static_cast<char>(toupper(in[0])); 86 else if (in[i] == '_' && i + 1 < in.length()) 87 s += static_cast<char>(toupper(in[++i])); 88 else 89 s += in[i]; 90 } 91 return s; 92 } 93 94 void DeserializeDoc( std::vector<std::string> &doc, 95 const Vector<Offset<String>> *documentation) { 96 if (documentation == nullptr) return; 97 for (uoffset_t index = 0; index < documentation->size(); index++) 98 doc.push_back(documentation->Get(index)->str()); 99 } 100 101 void Parser::Message(const std::string &msg) { 102 error_ = file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : ""; 103 // clang-format off 104 #ifdef _WIN32 // MSVC alike 105 error_ += 106 "(" + NumToString(line_) + ", " + NumToString(CursorPosition()) + ")"; 107 #else // gcc alike 108 if (file_being_parsed_.length()) error_ += ":"; 109 error_ += NumToString(line_) + ": " + NumToString(CursorPosition()); 110 #endif 111 // clang-format on 112 error_ += ": " + msg; 113 } 114 115 void Parser::Warning(const std::string &msg) { Message("warning: " + msg); } 116 117 CheckedError Parser::Error(const std::string &msg) { 118 Message("error: " + msg); 119 return CheckedError(true); 120 } 121 122 inline CheckedError NoError() { return CheckedError(false); } 123 124 CheckedError Parser::RecurseError() { 125 return Error("maximum parsing recursion of " + 126 NumToString(FLATBUFFERS_MAX_PARSING_DEPTH) + " reached"); 127 } 128 129 template<typename F> CheckedError Parser::Recurse(F f) { 130 if (recurse_protection_counter >= (FLATBUFFERS_MAX_PARSING_DEPTH)) 131 return RecurseError(); 132 recurse_protection_counter++; 133 auto ce = f(); 134 recurse_protection_counter--; 135 return ce; 136 } 137 138 CheckedError Parser::InvalidNumber(const char *number, const std::string &msg) { 139 return Error("invalid number: \"" + std::string(number) + "\"" + msg); 140 } 141 // atot: templated version of atoi/atof: convert a string to an instance of T. 142 template<typename T> 143 inline CheckedError atot(const char *s, Parser &parser, T *val) { 144 auto done = StringToNumber(s, val); 145 if (done) return NoError(); 146 147 return parser.InvalidNumber( 148 s, (0 == *val) 149 ? "" 150 : (", constant does not fit [" + 151 NumToString(flatbuffers::numeric_limits<T>::lowest()) + "; " + 152 NumToString(flatbuffers::numeric_limits<T>::max()) + "]")); 153 } 154 template<> 155 inline CheckedError atot<Offset<void>>(const char *s, Parser &parser, 156 Offset<void> *val) { 157 (void)parser; 158 *val = Offset<void>(atoi(s)); 159 return NoError(); 160 } 161 162 std::string Namespace::GetFullyQualifiedName(const std::string &name, 163 size_t max_components) const { 164 // Early exit if we don't have a defined namespace. 165 if (components.empty() || !max_components) { return name; } 166 std::string stream_str; 167 for (size_t i = 0; i < std::min(components.size(), max_components); i++) { 168 if (i) { stream_str += '.'; } 169 stream_str += std::string(components[i]); 170 } 171 if (name.length()) { 172 stream_str += '.'; 173 stream_str += name; 174 } 175 return stream_str; 176 } 177 178 // Declare tokens we'll use. Single character tokens are represented by their 179 // ascii character code (e.g. '{'), others above 256. 180 // clang-format off 181 #define FLATBUFFERS_GEN_TOKENS(TD) \ 182 TD(Eof, 256, "end of file") \ 183 TD(StringConstant, 257, "string constant") \ 184 TD(IntegerConstant, 258, "integer constant") \ 185 TD(FloatConstant, 259, "float constant") \ 186 TD(Identifier, 260, "identifier") 187 #ifdef __GNUC__ 188 __extension__ // Stop GCC complaining about trailing comma with -Wpendantic. 189 #endif 190 enum { 191 #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) kToken ## NAME = VALUE, 192 FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN) 193 #undef FLATBUFFERS_TOKEN 194 }; 195 196 static std::string TokenToString(int t) { 197 static const char * const tokens[] = { 198 #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) STRING, 199 FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN) 200 #undef FLATBUFFERS_TOKEN 201 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ 202 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ 203 IDLTYPE, 204 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) 205 #undef FLATBUFFERS_TD 206 }; 207 if (t < 256) { // A single ascii char token. 208 std::string s; 209 s.append(1, static_cast<char>(t)); 210 return s; 211 } else { // Other tokens. 212 return tokens[t - 256]; 213 } 214 } 215 // clang-format on 216 217 std::string Parser::TokenToStringId(int t) const { 218 return t == kTokenIdentifier ? attribute_ : TokenToString(t); 219 } 220 221 // Parses exactly nibbles worth of hex digits into a number, or error. 222 CheckedError Parser::ParseHexNum(int nibbles, uint64_t *val) { 223 FLATBUFFERS_ASSERT(nibbles > 0); 224 for (int i = 0; i < nibbles; i++) 225 if (!is_xdigit(cursor_[i])) 226 return Error("escape code must be followed by " + NumToString(nibbles) + 227 " hex digits"); 228 std::string target(cursor_, cursor_ + nibbles); 229 *val = StringToUInt(target.c_str(), 16); 230 cursor_ += nibbles; 231 return NoError(); 232 } 233 234 CheckedError Parser::SkipByteOrderMark() { 235 if (static_cast<unsigned char>(*cursor_) != 0xef) return NoError(); 236 cursor_++; 237 if (static_cast<unsigned char>(*cursor_) != 0xbb) 238 return Error("invalid utf-8 byte order mark"); 239 cursor_++; 240 if (static_cast<unsigned char>(*cursor_) != 0xbf) 241 return Error("invalid utf-8 byte order mark"); 242 cursor_++; 243 return NoError(); 244 } 245 246 static inline bool IsIdentifierStart(char c) { 247 return is_alpha(c) || (c == '_'); 248 } 249 250 CheckedError Parser::Next() { 251 doc_comment_.clear(); 252 bool seen_newline = cursor_ == source_; 253 attribute_.clear(); 254 attr_is_trivial_ascii_string_ = true; 255 for (;;) { 256 char c = *cursor_++; 257 token_ = c; 258 switch (c) { 259 case '\0': 260 cursor_--; 261 token_ = kTokenEof; 262 return NoError(); 263 case ' ': 264 case '\r': 265 case '\t': break; 266 case '\n': 267 MarkNewLine(); 268 seen_newline = true; 269 break; 270 case '{': 271 case '}': 272 case '(': 273 case ')': 274 case '[': 275 case ']': 276 case ',': 277 case ':': 278 case ';': 279 case '=': return NoError(); 280 case '\"': 281 case '\'': { 282 int unicode_high_surrogate = -1; 283 284 while (*cursor_ != c) { 285 if (*cursor_ < ' ' && static_cast<signed char>(*cursor_) >= 0) 286 return Error("illegal character in string constant"); 287 if (*cursor_ == '\\') { 288 attr_is_trivial_ascii_string_ = false; // has escape sequence 289 cursor_++; 290 if (unicode_high_surrogate != -1 && *cursor_ != 'u') { 291 return Error( 292 "illegal Unicode sequence (unpaired high surrogate)"); 293 } 294 switch (*cursor_) { 295 case 'n': 296 attribute_ += '\n'; 297 cursor_++; 298 break; 299 case 't': 300 attribute_ += '\t'; 301 cursor_++; 302 break; 303 case 'r': 304 attribute_ += '\r'; 305 cursor_++; 306 break; 307 case 'b': 308 attribute_ += '\b'; 309 cursor_++; 310 break; 311 case 'f': 312 attribute_ += '\f'; 313 cursor_++; 314 break; 315 case '\"': 316 attribute_ += '\"'; 317 cursor_++; 318 break; 319 case '\'': 320 attribute_ += '\''; 321 cursor_++; 322 break; 323 case '\\': 324 attribute_ += '\\'; 325 cursor_++; 326 break; 327 case '/': 328 attribute_ += '/'; 329 cursor_++; 330 break; 331 case 'x': { // Not in the JSON standard 332 cursor_++; 333 uint64_t val; 334 ECHECK(ParseHexNum(2, &val)); 335 attribute_ += static_cast<char>(val); 336 break; 337 } 338 case 'u': { 339 cursor_++; 340 uint64_t val; 341 ECHECK(ParseHexNum(4, &val)); 342 if (val >= 0xD800 && val <= 0xDBFF) { 343 if (unicode_high_surrogate != -1) { 344 return Error( 345 "illegal Unicode sequence (multiple high surrogates)"); 346 } else { 347 unicode_high_surrogate = static_cast<int>(val); 348 } 349 } else if (val >= 0xDC00 && val <= 0xDFFF) { 350 if (unicode_high_surrogate == -1) { 351 return Error( 352 "illegal Unicode sequence (unpaired low surrogate)"); 353 } else { 354 int code_point = 0x10000 + 355 ((unicode_high_surrogate & 0x03FF) << 10) + 356 (val & 0x03FF); 357 ToUTF8(code_point, &attribute_); 358 unicode_high_surrogate = -1; 359 } 360 } else { 361 if (unicode_high_surrogate != -1) { 362 return Error( 363 "illegal Unicode sequence (unpaired high surrogate)"); 364 } 365 ToUTF8(static_cast<int>(val), &attribute_); 366 } 367 break; 368 } 369 default: return Error("unknown escape code in string constant"); 370 } 371 } else { // printable chars + UTF-8 bytes 372 if (unicode_high_surrogate != -1) { 373 return Error( 374 "illegal Unicode sequence (unpaired high surrogate)"); 375 } 376 // reset if non-printable 377 attr_is_trivial_ascii_string_ &= check_in_range(*cursor_, ' ', '~'); 378 379 attribute_ += *cursor_++; 380 } 381 } 382 if (unicode_high_surrogate != -1) { 383 return Error("illegal Unicode sequence (unpaired high surrogate)"); 384 } 385 cursor_++; 386 if (!attr_is_trivial_ascii_string_ && !opts.allow_non_utf8 && 387 !ValidateUTF8(attribute_)) { 388 return Error("illegal UTF-8 sequence"); 389 } 390 token_ = kTokenStringConstant; 391 return NoError(); 392 } 393 case '/': 394 if (*cursor_ == '/') { 395 const char *start = ++cursor_; 396 while (*cursor_ && *cursor_ != '\n' && *cursor_ != '\r') cursor_++; 397 if (*start == '/') { // documentation comment 398 if (!seen_newline) 399 return Error( 400 "a documentation comment should be on a line on its own"); 401 doc_comment_.push_back(std::string(start + 1, cursor_)); 402 } 403 break; 404 } else if (*cursor_ == '*') { 405 cursor_++; 406 // TODO: make nested. 407 while (*cursor_ != '*' || cursor_[1] != '/') { 408 if (*cursor_ == '\n') MarkNewLine(); 409 if (!*cursor_) return Error("end of file in comment"); 410 cursor_++; 411 } 412 cursor_ += 2; 413 break; 414 } 415 FLATBUFFERS_FALLTHROUGH(); // else fall thru 416 default: 417 const auto has_sign = (c == '+') || (c == '-'); 418 // '-'/'+' and following identifier - can be a predefined constant like: 419 // NAN, INF, PI, etc. 420 if (IsIdentifierStart(c) || (has_sign && IsIdentifierStart(*cursor_))) { 421 // Collect all chars of an identifier: 422 const char *start = cursor_ - 1; 423 while (IsIdentifierStart(*cursor_) || is_digit(*cursor_)) cursor_++; 424 attribute_.append(start, cursor_); 425 token_ = has_sign ? kTokenStringConstant : kTokenIdentifier; 426 return NoError(); 427 } 428 429 auto dot_lvl = (c == '.') ? 0 : 1; // dot_lvl==0 <=> exactly one '.' seen 430 if (!dot_lvl && !is_digit(*cursor_)) return NoError(); // enum? 431 // Parser accepts hexadecimal-floating-literal (see C++ 5.13.4). 432 if (is_digit(c) || has_sign || !dot_lvl) { 433 const auto start = cursor_ - 1; 434 auto start_digits = !is_digit(c) ? cursor_ : cursor_ - 1; 435 if (!is_digit(c) && is_digit(*cursor_)){ 436 start_digits = cursor_; // see digit in cursor_ position 437 c = *cursor_++; 438 } 439 // hex-float can't begind with '.' 440 auto use_hex = dot_lvl && (c == '0') && is_alpha_char(*cursor_, 'X'); 441 if (use_hex) start_digits = ++cursor_; // '0x' is the prefix, skip it 442 // Read an integer number or mantisa of float-point number. 443 do { 444 if (use_hex) { 445 while (is_xdigit(*cursor_)) cursor_++; 446 } else { 447 while (is_digit(*cursor_)) cursor_++; 448 } 449 } while ((*cursor_ == '.') && (++cursor_) && (--dot_lvl >= 0)); 450 // Exponent of float-point number. 451 if ((dot_lvl >= 0) && (cursor_ > start_digits)) { 452 // The exponent suffix of hexadecimal float number is mandatory. 453 if (use_hex && !dot_lvl) start_digits = cursor_; 454 if ((use_hex && is_alpha_char(*cursor_, 'P')) || 455 is_alpha_char(*cursor_, 'E')) { 456 dot_lvl = 0; // Emulate dot to signal about float-point number. 457 cursor_++; 458 if (*cursor_ == '+' || *cursor_ == '-') cursor_++; 459 start_digits = cursor_; // the exponent-part has to have digits 460 // Exponent is decimal integer number 461 while (is_digit(*cursor_)) cursor_++; 462 if (*cursor_ == '.') { 463 cursor_++; // If see a dot treat it as part of invalid number. 464 dot_lvl = -1; // Fall thru to Error(). 465 } 466 } 467 } 468 // Finalize. 469 if ((dot_lvl >= 0) && (cursor_ > start_digits)) { 470 attribute_.append(start, cursor_); 471 token_ = dot_lvl ? kTokenIntegerConstant : kTokenFloatConstant; 472 return NoError(); 473 } else { 474 return Error("invalid number: " + std::string(start, cursor_)); 475 } 476 } 477 std::string ch; 478 ch = c; 479 if (false == check_in_range(c, ' ', '~')) ch = "code: " + NumToString(c); 480 return Error("illegal character: " + ch); 481 } 482 } 483 } 484 485 // Check if a given token is next. 486 bool Parser::Is(int t) const { return t == token_; } 487 488 bool Parser::IsIdent(const char *id) const { 489 return token_ == kTokenIdentifier && attribute_ == id; 490 } 491 492 // Expect a given token to be next, consume it, or error if not present. 493 CheckedError Parser::Expect(int t) { 494 if (t != token_) { 495 return Error("expecting: " + TokenToString(t) + 496 " instead got: " + TokenToStringId(token_)); 497 } 498 NEXT(); 499 return NoError(); 500 } 501 502 CheckedError Parser::ParseNamespacing(std::string *id, std::string *last) { 503 while (Is('.')) { 504 NEXT(); 505 *id += "."; 506 *id += attribute_; 507 if (last) *last = attribute_; 508 EXPECT(kTokenIdentifier); 509 } 510 return NoError(); 511 } 512 513 EnumDef *Parser::LookupEnum(const std::string &id) { 514 // Search thru parent namespaces. 515 for (int components = static_cast<int>(current_namespace_->components.size()); 516 components >= 0; components--) { 517 auto ed = enums_.Lookup( 518 current_namespace_->GetFullyQualifiedName(id, components)); 519 if (ed) return ed; 520 } 521 return nullptr; 522 } 523 524 StructDef *Parser::LookupStruct(const std::string &id) const { 525 auto sd = structs_.Lookup(id); 526 if (sd) sd->refcount++; 527 return sd; 528 } 529 530 CheckedError Parser::ParseTypeIdent(Type &type) { 531 std::string id = attribute_; 532 EXPECT(kTokenIdentifier); 533 ECHECK(ParseNamespacing(&id, nullptr)); 534 auto enum_def = LookupEnum(id); 535 if (enum_def) { 536 type = enum_def->underlying_type; 537 if (enum_def->is_union) type.base_type = BASE_TYPE_UNION; 538 } else { 539 type.base_type = BASE_TYPE_STRUCT; 540 type.struct_def = LookupCreateStruct(id); 541 } 542 return NoError(); 543 } 544 545 // Parse any IDL type. 546 CheckedError Parser::ParseType(Type &type) { 547 if (token_ == kTokenIdentifier) { 548 if (IsIdent("bool")) { 549 type.base_type = BASE_TYPE_BOOL; 550 NEXT(); 551 } else if (IsIdent("byte") || IsIdent("int8")) { 552 type.base_type = BASE_TYPE_CHAR; 553 NEXT(); 554 } else if (IsIdent("ubyte") || IsIdent("uint8")) { 555 type.base_type = BASE_TYPE_UCHAR; 556 NEXT(); 557 } else if (IsIdent("short") || IsIdent("int16")) { 558 type.base_type = BASE_TYPE_SHORT; 559 NEXT(); 560 } else if (IsIdent("ushort") || IsIdent("uint16")) { 561 type.base_type = BASE_TYPE_USHORT; 562 NEXT(); 563 } else if (IsIdent("int") || IsIdent("int32")) { 564 type.base_type = BASE_TYPE_INT; 565 NEXT(); 566 } else if (IsIdent("uint") || IsIdent("uint32")) { 567 type.base_type = BASE_TYPE_UINT; 568 NEXT(); 569 } else if (IsIdent("long") || IsIdent("int64")) { 570 type.base_type = BASE_TYPE_LONG; 571 NEXT(); 572 } else if (IsIdent("ulong") || IsIdent("uint64")) { 573 type.base_type = BASE_TYPE_ULONG; 574 NEXT(); 575 } else if (IsIdent("float") || IsIdent("float32")) { 576 type.base_type = BASE_TYPE_FLOAT; 577 NEXT(); 578 } else if (IsIdent("double") || IsIdent("float64")) { 579 type.base_type = BASE_TYPE_DOUBLE; 580 NEXT(); 581 } else if (IsIdent("string")) { 582 type.base_type = BASE_TYPE_STRING; 583 NEXT(); 584 } else { 585 ECHECK(ParseTypeIdent(type)); 586 } 587 } else if (token_ == '[') { 588 NEXT(); 589 Type subtype; 590 ECHECK(Recurse([&]() { return ParseType(subtype); })); 591 if (subtype.base_type == BASE_TYPE_VECTOR) { 592 // We could support this, but it will complicate things, and it's 593 // easier to work around with a struct around the inner vector. 594 return Error("nested vector types not supported (wrap in table first)."); 595 } 596 type = Type(BASE_TYPE_VECTOR, subtype.struct_def, subtype.enum_def); 597 type.element = subtype.base_type; 598 EXPECT(']'); 599 } else { 600 return Error("illegal type syntax"); 601 } 602 return NoError(); 603 } 604 605 CheckedError Parser::AddField(StructDef &struct_def, const std::string &name, 606 const Type &type, FieldDef **dest) { 607 auto &field = *new FieldDef(); 608 field.value.offset = 609 FieldIndexToOffset(static_cast<voffset_t>(struct_def.fields.vec.size())); 610 field.name = name; 611 field.file = struct_def.file; 612 field.value.type = type; 613 if (struct_def.fixed) { // statically compute the field offset 614 auto size = InlineSize(type); 615 auto alignment = InlineAlignment(type); 616 // structs_ need to have a predictable format, so we need to align to 617 // the largest scalar 618 struct_def.minalign = std::max(struct_def.minalign, alignment); 619 struct_def.PadLastField(alignment); 620 field.value.offset = static_cast<voffset_t>(struct_def.bytesize); 621 struct_def.bytesize += size; 622 } 623 if (struct_def.fields.Add(name, &field)) 624 return Error("field already exists: " + name); 625 *dest = &field; 626 return NoError(); 627 } 628 629 CheckedError Parser::ParseField(StructDef &struct_def) { 630 std::string name = attribute_; 631 632 if (LookupStruct(name)) 633 return Error("field name can not be the same as table/struct name"); 634 635 std::vector<std::string> dc = doc_comment_; 636 EXPECT(kTokenIdentifier); 637 EXPECT(':'); 638 Type type; 639 ECHECK(ParseType(type)); 640 641 if (struct_def.fixed && !IsScalar(type.base_type) && !IsStruct(type)) 642 return Error("structs_ may contain only scalar or struct fields"); 643 644 FieldDef *typefield = nullptr; 645 if (type.base_type == BASE_TYPE_UNION) { 646 // For union fields, add a second auto-generated field to hold the type, 647 // with a special suffix. 648 ECHECK(AddField(struct_def, name + UnionTypeFieldSuffix(), 649 type.enum_def->underlying_type, &typefield)); 650 } else if (type.base_type == BASE_TYPE_VECTOR && 651 type.element == BASE_TYPE_UNION) { 652 // Only cpp, js and ts supports the union vector feature so far. 653 if (!SupportsVectorOfUnions()) { 654 return Error( 655 "Vectors of unions are not yet supported in all " 656 "the specified programming languages."); 657 } 658 // For vector of union fields, add a second auto-generated vector field to 659 // hold the types, with a special suffix. 660 Type union_vector(BASE_TYPE_VECTOR, nullptr, type.enum_def); 661 union_vector.element = BASE_TYPE_UTYPE; 662 ECHECK(AddField(struct_def, name + UnionTypeFieldSuffix(), union_vector, 663 &typefield)); 664 } 665 666 FieldDef *field; 667 ECHECK(AddField(struct_def, name, type, &field)); 668 669 if (token_ == '=') { 670 NEXT(); 671 if (!IsScalar(type.base_type) || 672 (struct_def.fixed && field->value.constant != "0")) 673 return Error( 674 "default values currently only supported for scalars in tables"); 675 ECHECK(ParseSingleValue(&field->name, field->value, true)); 676 } 677 if (type.enum_def && 678 !type.enum_def->is_union && 679 !type.enum_def->attributes.Lookup("bit_flags") && 680 !type.enum_def->ReverseLookup(StringToInt( 681 field->value.constant.c_str()))) { 682 return Error("default value of " + field->value.constant + " for field " + 683 name + " is not part of enum " + type.enum_def->name); 684 } 685 // Append .0 if the value has not it (skip hex and scientific floats). 686 // This suffix needed for generated C++ code. 687 if (IsFloat(type.base_type)) { 688 auto &text = field->value.constant; 689 FLATBUFFERS_ASSERT(false == text.empty()); 690 auto s = text.c_str(); 691 while(*s == ' ') s++; 692 if (*s == '-' || *s == '+') s++; 693 // 1) A float constants (nan, inf, pi, etc) is a kind of identifier. 694 // 2) A float number needn't ".0" at the end if it has exponent. 695 if ((false == IsIdentifierStart(*s)) && 696 (std::string::npos == field->value.constant.find_first_of(".eEpP"))) { 697 field->value.constant += ".0"; 698 } 699 } 700 701 if (type.enum_def && IsScalar(type.base_type) && !struct_def.fixed && 702 !type.enum_def->attributes.Lookup("bit_flags") && 703 !type.enum_def->ReverseLookup(StringToInt( 704 field->value.constant.c_str()))) 705 Warning("enum " + type.enum_def->name + 706 " does not have a declaration for this field\'s default of " + 707 field->value.constant); 708 709 field->doc_comment = dc; 710 ECHECK(ParseMetaData(&field->attributes)); 711 field->deprecated = field->attributes.Lookup("deprecated") != nullptr; 712 auto hash_name = field->attributes.Lookup("hash"); 713 if (hash_name) { 714 switch ((type.base_type == BASE_TYPE_VECTOR) ? type.element : type.base_type) { 715 case BASE_TYPE_SHORT: 716 case BASE_TYPE_USHORT: { 717 if (FindHashFunction16(hash_name->constant.c_str()) == nullptr) 718 return Error("Unknown hashing algorithm for 16 bit types: " + 719 hash_name->constant); 720 break; 721 } 722 case BASE_TYPE_INT: 723 case BASE_TYPE_UINT: { 724 if (FindHashFunction32(hash_name->constant.c_str()) == nullptr) 725 return Error("Unknown hashing algorithm for 32 bit types: " + 726 hash_name->constant); 727 break; 728 } 729 case BASE_TYPE_LONG: 730 case BASE_TYPE_ULONG: { 731 if (FindHashFunction64(hash_name->constant.c_str()) == nullptr) 732 return Error("Unknown hashing algorithm for 64 bit types: " + 733 hash_name->constant); 734 break; 735 } 736 default: 737 return Error( 738 "only short, ushort, int, uint, long and ulong data types support hashing."); 739 } 740 } 741 auto cpp_type = field->attributes.Lookup("cpp_type"); 742 if (cpp_type) { 743 if (!hash_name) 744 return Error("cpp_type can only be used with a hashed field"); 745 /// forcing cpp_ptr_type to 'naked' if unset 746 auto cpp_ptr_type = field->attributes.Lookup("cpp_ptr_type"); 747 if (!cpp_ptr_type) { 748 auto val = new Value(); 749 val->type = cpp_type->type; 750 val->constant = "naked"; 751 field->attributes.Add("cpp_ptr_type", val); 752 } 753 } 754 if (field->deprecated && struct_def.fixed) 755 return Error("can't deprecate fields in a struct"); 756 field->required = field->attributes.Lookup("required") != nullptr; 757 if (field->required && 758 (struct_def.fixed || IsScalar(type.base_type))) 759 return Error("only non-scalar fields in tables may be 'required'"); 760 field->key = field->attributes.Lookup("key") != nullptr; 761 if (field->key) { 762 if (struct_def.has_key) return Error("only one field may be set as 'key'"); 763 struct_def.has_key = true; 764 if (!IsScalar(type.base_type)) { 765 field->required = true; 766 if (type.base_type != BASE_TYPE_STRING) 767 return Error("'key' field must be string or scalar type"); 768 } 769 } 770 field->shared = field->attributes.Lookup("shared") != nullptr; 771 if (field->shared && field->value.type.base_type != BASE_TYPE_STRING) 772 return Error("shared can only be defined on strings"); 773 774 auto field_native_custom_alloc = 775 field->attributes.Lookup("native_custom_alloc"); 776 if (field_native_custom_alloc) 777 return Error( 778 "native_custom_alloc can only be used with a table or struct " 779 "definition"); 780 781 field->native_inline = field->attributes.Lookup("native_inline") != nullptr; 782 if (field->native_inline && !IsStruct(field->value.type)) 783 return Error("native_inline can only be defined on structs"); 784 785 auto nested = field->attributes.Lookup("nested_flatbuffer"); 786 if (nested) { 787 if (nested->type.base_type != BASE_TYPE_STRING) 788 return Error( 789 "nested_flatbuffer attribute must be a string (the root type)"); 790 if (type.base_type != BASE_TYPE_VECTOR || type.element != BASE_TYPE_UCHAR) 791 return Error( 792 "nested_flatbuffer attribute may only apply to a vector of ubyte"); 793 // This will cause an error if the root type of the nested flatbuffer 794 // wasn't defined elsewhere. 795 LookupCreateStruct(nested->constant); 796 797 // Keep a pointer to StructDef in FieldDef to simplify re-use later 798 auto nested_qualified_name = 799 current_namespace_->GetFullyQualifiedName(nested->constant); 800 field->nested_flatbuffer = LookupStruct(nested_qualified_name); 801 } 802 803 if (field->attributes.Lookup("flexbuffer")) { 804 field->flexbuffer = true; 805 uses_flexbuffers_ = true; 806 if (type.base_type != BASE_TYPE_VECTOR || 807 type.element != BASE_TYPE_UCHAR) 808 return Error("flexbuffer attribute may only apply to a vector of ubyte"); 809 } 810 811 if (typefield) { 812 if (!IsScalar(typefield->value.type.base_type)) { 813 // this is a union vector field 814 typefield->required = field->required; 815 } 816 // If this field is a union, and it has a manually assigned id, 817 // the automatically added type field should have an id as well (of N - 1). 818 auto attr = field->attributes.Lookup("id"); 819 if (attr) { 820 auto id = atoi(attr->constant.c_str()); 821 auto val = new Value(); 822 val->type = attr->type; 823 val->constant = NumToString(id - 1); 824 typefield->attributes.Add("id", val); 825 } 826 } 827 828 EXPECT(';'); 829 return NoError(); 830 } 831 832 CheckedError Parser::ParseString(Value &val) { 833 auto s = attribute_; 834 EXPECT(kTokenStringConstant); 835 val.constant = NumToString(builder_.CreateString(s).o); 836 return NoError(); 837 } 838 839 CheckedError Parser::ParseComma() { 840 if (!opts.protobuf_ascii_alike) EXPECT(','); 841 return NoError(); 842 } 843 844 CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field, 845 size_t parent_fieldn, 846 const StructDef *parent_struct_def) { 847 switch (val.type.base_type) { 848 case BASE_TYPE_UNION: { 849 FLATBUFFERS_ASSERT(field); 850 std::string constant; 851 // Find corresponding type field we may have already parsed. 852 for (auto elem = field_stack_.rbegin(); 853 elem != field_stack_.rbegin() + parent_fieldn; ++elem) { 854 auto &type = elem->second->value.type; 855 if (type.base_type == BASE_TYPE_UTYPE && 856 type.enum_def == val.type.enum_def) { 857 constant = elem->first.constant; 858 break; 859 } 860 } 861 if (constant.empty()) { 862 // We haven't seen the type field yet. Sadly a lot of JSON writers 863 // output these in alphabetical order, meaning it comes after this 864 // value. So we scan past the value to find it, then come back here. 865 auto type_name = field->name + UnionTypeFieldSuffix(); 866 FLATBUFFERS_ASSERT(parent_struct_def); 867 auto type_field = parent_struct_def->fields.Lookup(type_name); 868 FLATBUFFERS_ASSERT(type_field); // Guaranteed by ParseField(). 869 // Remember where we are in the source file, so we can come back here. 870 auto backup = *static_cast<ParserState *>(this); 871 ECHECK(SkipAnyJsonValue()); // The table. 872 ECHECK(ParseComma()); 873 auto next_name = attribute_; 874 if (Is(kTokenStringConstant)) { 875 NEXT(); 876 } else { 877 EXPECT(kTokenIdentifier); 878 } 879 if (next_name != type_name) 880 return Error("missing type field after this union value: " + 881 type_name); 882 EXPECT(':'); 883 Value type_val = type_field->value; 884 ECHECK(ParseAnyValue(type_val, type_field, 0, nullptr)); 885 constant = type_val.constant; 886 // Got the information we needed, now rewind: 887 *static_cast<ParserState *>(this) = backup; 888 } 889 uint8_t enum_idx; 890 ECHECK(atot(constant.c_str(), *this, &enum_idx)); 891 auto enum_val = val.type.enum_def->ReverseLookup(enum_idx); 892 if (!enum_val) return Error("illegal type id for: " + field->name); 893 if (enum_val->union_type.base_type == BASE_TYPE_STRUCT) { 894 ECHECK(ParseTable(*enum_val->union_type.struct_def, &val.constant, 895 nullptr)); 896 if (enum_val->union_type.struct_def->fixed) { 897 // All BASE_TYPE_UNION values are offsets, so turn this into one. 898 SerializeStruct(*enum_val->union_type.struct_def, val); 899 builder_.ClearOffsets(); 900 val.constant = NumToString(builder_.GetSize()); 901 } 902 } else if (enum_val->union_type.base_type == BASE_TYPE_STRING) { 903 ECHECK(ParseString(val)); 904 } else { 905 FLATBUFFERS_ASSERT(false); 906 } 907 break; 908 } 909 case BASE_TYPE_STRUCT: 910 ECHECK(ParseTable(*val.type.struct_def, &val.constant, nullptr)); 911 break; 912 case BASE_TYPE_STRING: { 913 ECHECK(ParseString(val)); 914 break; 915 } 916 case BASE_TYPE_VECTOR: { 917 uoffset_t off; 918 ECHECK(ParseVector(val.type.VectorType(), &off)); 919 val.constant = NumToString(off); 920 break; 921 } 922 case BASE_TYPE_INT: 923 case BASE_TYPE_UINT: 924 case BASE_TYPE_LONG: 925 case BASE_TYPE_ULONG: { 926 if (field && field->attributes.Lookup("hash") && 927 (token_ == kTokenIdentifier || token_ == kTokenStringConstant)) { 928 ECHECK(ParseHash(val, field)); 929 } else { 930 ECHECK(ParseSingleValue(field ? &field->name : nullptr, val, false)); 931 } 932 break; 933 } 934 default: 935 ECHECK(ParseSingleValue(field ? &field->name : nullptr, val, false)); 936 break; 937 } 938 return NoError(); 939 } 940 941 void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) { 942 FLATBUFFERS_ASSERT(val.constant.length() == struct_def.bytesize); 943 builder_.Align(struct_def.minalign); 944 builder_.PushBytes(reinterpret_cast<const uint8_t *>(val.constant.c_str()), 945 struct_def.bytesize); 946 builder_.AddStructOffset(val.offset, builder_.GetSize()); 947 } 948 949 template <typename F> 950 CheckedError Parser::ParseTableDelimiters(size_t &fieldn, 951 const StructDef *struct_def, 952 F body) { 953 // We allow tables both as JSON object{ .. } with field names 954 // or vector[..] with all fields in order 955 char terminator = '}'; 956 bool is_nested_vector = struct_def && Is('['); 957 if (is_nested_vector) { 958 NEXT(); 959 terminator = ']'; 960 } else { 961 EXPECT('{'); 962 } 963 for (;;) { 964 if ((!opts.strict_json || !fieldn) && Is(terminator)) break; 965 std::string name; 966 if (is_nested_vector) { 967 if (fieldn >= struct_def->fields.vec.size()) { 968 return Error("too many unnamed fields in nested array"); 969 } 970 name = struct_def->fields.vec[fieldn]->name; 971 } else { 972 name = attribute_; 973 if (Is(kTokenStringConstant)) { 974 NEXT(); 975 } else { 976 EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier); 977 } 978 if (!opts.protobuf_ascii_alike || !(Is('{') || Is('['))) EXPECT(':'); 979 } 980 ECHECK(body(name, fieldn, struct_def)); 981 if (Is(terminator)) break; 982 ECHECK(ParseComma()); 983 } 984 NEXT(); 985 if (is_nested_vector && fieldn != struct_def->fields.vec.size()) { 986 return Error("wrong number of unnamed fields in table vector"); 987 } 988 return NoError(); 989 } 990 991 CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, 992 uoffset_t *ovalue) { 993 size_t fieldn_outer = 0; 994 auto err = ParseTableDelimiters( 995 fieldn_outer, &struct_def, 996 [&](const std::string &name, size_t &fieldn, 997 const StructDef *struct_def_inner) -> CheckedError { 998 if (name == "$schema") { 999 ECHECK(Expect(kTokenStringConstant)); 1000 return NoError(); 1001 } 1002 auto field = struct_def_inner->fields.Lookup(name); 1003 if (!field) { 1004 if (!opts.skip_unexpected_fields_in_json) { 1005 return Error("unknown field: " + name); 1006 } else { 1007 ECHECK(SkipAnyJsonValue()); 1008 } 1009 } else { 1010 if (IsIdent("null") && !IsScalar(field->value.type.base_type)) { 1011 ECHECK(Next()); // Ignore this field. 1012 } else { 1013 Value val = field->value; 1014 if (field->flexbuffer) { 1015 flexbuffers::Builder builder(1024, 1016 flexbuffers::BUILDER_FLAG_SHARE_ALL); 1017 ECHECK(ParseFlexBufferValue(&builder)); 1018 builder.Finish(); 1019 // Force alignment for nested flexbuffer 1020 builder_.ForceVectorAlignment(builder.GetSize(), sizeof(uint8_t), 1021 sizeof(largest_scalar_t)); 1022 auto off = builder_.CreateVector(builder.GetBuffer()); 1023 val.constant = NumToString(off.o); 1024 } else if (field->nested_flatbuffer) { 1025 ECHECK( 1026 ParseNestedFlatbuffer(val, field, fieldn, struct_def_inner)); 1027 } else { 1028 ECHECK(Recurse([&]() { 1029 return ParseAnyValue(val, field, fieldn, struct_def_inner); 1030 })); 1031 } 1032 // Hardcoded insertion-sort with error-check. 1033 // If fields are specified in order, then this loop exits 1034 // immediately. 1035 auto elem = field_stack_.rbegin(); 1036 for (; elem != field_stack_.rbegin() + fieldn; ++elem) { 1037 auto existing_field = elem->second; 1038 if (existing_field == field) 1039 return Error("field set more than once: " + field->name); 1040 if (existing_field->value.offset < field->value.offset) break; 1041 } 1042 // Note: elem points to before the insertion point, thus .base() 1043 // points to the correct spot. 1044 field_stack_.insert(elem.base(), std::make_pair(val, field)); 1045 fieldn++; 1046 } 1047 } 1048 return NoError(); 1049 }); 1050 ECHECK(err); 1051 1052 // Check if all required fields are parsed. 1053 for (auto field_it = struct_def.fields.vec.begin(); 1054 field_it != struct_def.fields.vec.end(); ++field_it) { 1055 auto required_field = *field_it; 1056 if (!required_field->required) { continue; } 1057 bool found = false; 1058 for (auto pf_it = field_stack_.end() - fieldn_outer; 1059 pf_it != field_stack_.end(); ++pf_it) { 1060 auto parsed_field = pf_it->second; 1061 if (parsed_field == required_field) { 1062 found = true; 1063 break; 1064 } 1065 } 1066 if (!found) { 1067 return Error("required field is missing: " + required_field->name + 1068 " in " + struct_def.name); 1069 } 1070 } 1071 1072 if (struct_def.fixed && fieldn_outer != struct_def.fields.vec.size()) 1073 return Error("struct: wrong number of initializers: " + struct_def.name); 1074 1075 auto start = struct_def.fixed ? builder_.StartStruct(struct_def.minalign) 1076 : builder_.StartTable(); 1077 1078 for (size_t size = struct_def.sortbysize ? sizeof(largest_scalar_t) : 1; size; 1079 size /= 2) { 1080 // Go through elements in reverse, since we're building the data backwards. 1081 for (auto it = field_stack_.rbegin(); 1082 it != field_stack_.rbegin() + fieldn_outer; ++it) { 1083 auto &field_value = it->first; 1084 auto field = it->second; 1085 if (!struct_def.sortbysize || 1086 size == SizeOf(field_value.type.base_type)) { 1087 switch (field_value.type.base_type) { 1088 // clang-format off 1089 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ 1090 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ 1091 case BASE_TYPE_ ## ENUM: \ 1092 builder_.Pad(field->padding); \ 1093 if (struct_def.fixed) { \ 1094 CTYPE val; \ 1095 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ 1096 builder_.PushElement(val); \ 1097 } else { \ 1098 CTYPE val, valdef; \ 1099 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ 1100 ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \ 1101 builder_.AddElement(field_value.offset, val, valdef); \ 1102 } \ 1103 break; 1104 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD); 1105 #undef FLATBUFFERS_TD 1106 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ 1107 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ 1108 case BASE_TYPE_ ## ENUM: \ 1109 builder_.Pad(field->padding); \ 1110 if (IsStruct(field->value.type)) { \ 1111 SerializeStruct(*field->value.type.struct_def, field_value); \ 1112 } else { \ 1113 CTYPE val; \ 1114 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ 1115 builder_.AddOffset(field_value.offset, val); \ 1116 } \ 1117 break; 1118 FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD); 1119 #undef FLATBUFFERS_TD 1120 // clang-format on 1121 } 1122 } 1123 } 1124 } 1125 for (size_t i = 0; i < fieldn_outer; i++) field_stack_.pop_back(); 1126 1127 if (struct_def.fixed) { 1128 builder_.ClearOffsets(); 1129 builder_.EndStruct(); 1130 FLATBUFFERS_ASSERT(value); 1131 // Temporarily store this struct in the value string, since it is to 1132 // be serialized in-place elsewhere. 1133 value->assign( 1134 reinterpret_cast<const char *>(builder_.GetCurrentBufferPointer()), 1135 struct_def.bytesize); 1136 builder_.PopBytes(struct_def.bytesize); 1137 FLATBUFFERS_ASSERT(!ovalue); 1138 } else { 1139 auto val = builder_.EndTable(start); 1140 if (ovalue) *ovalue = val; 1141 if (value) *value = NumToString(val); 1142 } 1143 return NoError(); 1144 } 1145 1146 template <typename F> 1147 CheckedError Parser::ParseVectorDelimiters(size_t &count, F body) { 1148 EXPECT('['); 1149 for (;;) { 1150 if ((!opts.strict_json || !count) && Is(']')) break; 1151 ECHECK(body(count)); 1152 count++; 1153 if (Is(']')) break; 1154 ECHECK(ParseComma()); 1155 } 1156 NEXT(); 1157 return NoError(); 1158 } 1159 1160 CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) { 1161 size_t count = 0; 1162 auto err = ParseVectorDelimiters(count, [&](size_t &) -> CheckedError { 1163 Value val; 1164 val.type = type; 1165 ECHECK(Recurse([&]() { return ParseAnyValue(val, nullptr, 0, nullptr); })); 1166 field_stack_.push_back(std::make_pair(val, nullptr)); 1167 return NoError(); 1168 }); 1169 ECHECK(err); 1170 1171 builder_.StartVector(count * InlineSize(type) / InlineAlignment(type), 1172 InlineAlignment(type)); 1173 for (size_t i = 0; i < count; i++) { 1174 // start at the back, since we're building the data backwards. 1175 auto &val = field_stack_.back().first; 1176 switch (val.type.base_type) { 1177 // clang-format off 1178 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ 1179 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ 1180 case BASE_TYPE_ ## ENUM: \ 1181 if (IsStruct(val.type)) SerializeStruct(*val.type.struct_def, val); \ 1182 else { \ 1183 CTYPE elem; \ 1184 ECHECK(atot(val.constant.c_str(), *this, &elem)); \ 1185 builder_.PushElement(elem); \ 1186 } \ 1187 break; 1188 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) 1189 #undef FLATBUFFERS_TD 1190 // clang-format on 1191 } 1192 field_stack_.pop_back(); 1193 } 1194 1195 builder_.ClearOffsets(); 1196 *ovalue = builder_.EndVector(count); 1197 return NoError(); 1198 } 1199 1200 CheckedError Parser::ParseNestedFlatbuffer(Value &val, FieldDef *field, 1201 size_t fieldn, 1202 const StructDef *parent_struct_def) { 1203 if (token_ == '[') { // backwards compat for 'legacy' ubyte buffers 1204 ECHECK(ParseAnyValue(val, field, fieldn, parent_struct_def)); 1205 } else { 1206 auto cursor_at_value_begin = cursor_; 1207 ECHECK(SkipAnyJsonValue()); 1208 std::string substring(cursor_at_value_begin - 1, cursor_ - 1); 1209 1210 // Create and initialize new parser 1211 Parser nested_parser; 1212 FLATBUFFERS_ASSERT(field->nested_flatbuffer); 1213 nested_parser.root_struct_def_ = field->nested_flatbuffer; 1214 nested_parser.enums_ = enums_; 1215 nested_parser.opts = opts; 1216 nested_parser.uses_flexbuffers_ = uses_flexbuffers_; 1217 1218 // Parse JSON substring into new flatbuffer builder using nested_parser 1219 bool ok = nested_parser.Parse(substring.c_str(), nullptr, nullptr); 1220 1221 // Clean nested_parser to avoid deleting the elements in 1222 // the SymbolTables on destruction 1223 nested_parser.enums_.dict.clear(); 1224 nested_parser.enums_.vec.clear(); 1225 1226 if (!ok) { 1227 ECHECK(Error(nested_parser.error_)); 1228 } 1229 // Force alignment for nested flatbuffer 1230 builder_.ForceVectorAlignment(nested_parser.builder_.GetSize(), sizeof(uint8_t), 1231 nested_parser.builder_.GetBufferMinAlignment()); 1232 1233 auto off = builder_.CreateVector(nested_parser.builder_.GetBufferPointer(), 1234 nested_parser.builder_.GetSize()); 1235 val.constant = NumToString(off.o); 1236 } 1237 return NoError(); 1238 } 1239 1240 CheckedError Parser::ParseMetaData(SymbolTable<Value> *attributes) { 1241 if (Is('(')) { 1242 NEXT(); 1243 for (;;) { 1244 auto name = attribute_; 1245 if (false == (Is(kTokenIdentifier) || Is(kTokenStringConstant))) 1246 return Error("attribute name must be either identifier or string: " + 1247 name); 1248 if (known_attributes_.find(name) == known_attributes_.end()) 1249 return Error("user define attributes must be declared before use: " + 1250 name); 1251 NEXT(); 1252 auto e = new Value(); 1253 attributes->Add(name, e); 1254 if (Is(':')) { 1255 NEXT(); 1256 ECHECK(ParseSingleValue(&name, *e, true)); 1257 } 1258 if (Is(')')) { 1259 NEXT(); 1260 break; 1261 } 1262 EXPECT(','); 1263 } 1264 } 1265 return NoError(); 1266 } 1267 1268 CheckedError Parser::TryTypedValue(const std::string *name, int dtoken, 1269 bool check, Value &e, BaseType req, 1270 bool *destmatch) { 1271 bool match = dtoken == token_; 1272 if (match) { 1273 FLATBUFFERS_ASSERT(*destmatch == false); 1274 *destmatch = true; 1275 e.constant = attribute_; 1276 // Check token match 1277 if (!check) { 1278 if (e.type.base_type == BASE_TYPE_NONE) { 1279 e.type.base_type = req; 1280 } else { 1281 return Error( 1282 std::string("type mismatch: expecting: ") + 1283 kTypeNames[e.type.base_type] + ", found: " + kTypeNames[req] + 1284 ", name: " + (name ? *name : "") + ", value: " + e.constant); 1285 } 1286 } 1287 // The exponent suffix of hexadecimal float-point number is mandatory. 1288 // A hex-integer constant is forbidden as an initializer of float number. 1289 if ((kTokenFloatConstant != dtoken) && IsFloat(e.type.base_type)) { 1290 const auto &s = e.constant; 1291 const auto k = s.find_first_of("0123456789."); 1292 if ((std::string::npos != k) && (s.length() > (k + 1)) && 1293 (s.at(k) == '0' && is_alpha_char(s.at(k + 1), 'X')) && 1294 (std::string::npos == s.find_first_of("pP", k + 2))) { 1295 return Error( 1296 "invalid number, the exponent suffix of hexadecimal " 1297 "floating-point literals is mandatory: \"" + 1298 s + "\""); 1299 } 1300 } 1301 1302 NEXT(); 1303 } 1304 return NoError(); 1305 } 1306 1307 CheckedError Parser::ParseEnumFromString(Type &type, int64_t *result) { 1308 *result = 0; 1309 // Parse one or more enum identifiers, separated by spaces. 1310 const char *next = attribute_.c_str(); 1311 do { 1312 const char *divider = strchr(next, ' '); 1313 std::string word; 1314 if (divider) { 1315 word = std::string(next, divider); 1316 next = divider + strspn(divider, " "); 1317 } else { 1318 word = next; 1319 next += word.length(); 1320 } 1321 if (type.enum_def) { // The field has an enum type 1322 auto enum_val = type.enum_def->vals.Lookup(word); 1323 if (!enum_val) 1324 return Error("unknown enum value: " + word + 1325 ", for enum: " + type.enum_def->name); 1326 *result |= enum_val->value; 1327 } else { // No enum type, probably integral field. 1328 if (!IsInteger(type.base_type)) 1329 return Error("not a valid value for this field: " + word); 1330 // TODO: could check if its a valid number constant here. 1331 const char *dot = strrchr(word.c_str(), '.'); 1332 if (!dot) 1333 return Error("enum values need to be qualified by an enum type"); 1334 std::string enum_def_str(word.c_str(), dot); 1335 std::string enum_val_str(dot + 1, word.c_str() + word.length()); 1336 auto enum_def = LookupEnum(enum_def_str); 1337 if (!enum_def) return Error("unknown enum: " + enum_def_str); 1338 auto enum_val = enum_def->vals.Lookup(enum_val_str); 1339 if (!enum_val) return Error("unknown enum value: " + enum_val_str); 1340 *result |= enum_val->value; 1341 } 1342 } while (*next); 1343 return NoError(); 1344 } 1345 1346 CheckedError Parser::ParseHash(Value &e, FieldDef *field) { 1347 FLATBUFFERS_ASSERT(field); 1348 Value *hash_name = field->attributes.Lookup("hash"); 1349 switch (e.type.base_type) { 1350 case BASE_TYPE_SHORT: { 1351 auto hash = FindHashFunction16(hash_name->constant.c_str()); 1352 int16_t hashed_value = static_cast<int16_t>(hash(attribute_.c_str())); 1353 e.constant = NumToString(hashed_value); 1354 break; 1355 } 1356 case BASE_TYPE_USHORT: { 1357 auto hash = FindHashFunction16(hash_name->constant.c_str()); 1358 uint16_t hashed_value = hash(attribute_.c_str()); 1359 e.constant = NumToString(hashed_value); 1360 break; 1361 } 1362 case BASE_TYPE_INT: { 1363 auto hash = FindHashFunction32(hash_name->constant.c_str()); 1364 int32_t hashed_value = static_cast<int32_t>(hash(attribute_.c_str())); 1365 e.constant = NumToString(hashed_value); 1366 break; 1367 } 1368 case BASE_TYPE_UINT: { 1369 auto hash = FindHashFunction32(hash_name->constant.c_str()); 1370 uint32_t hashed_value = hash(attribute_.c_str()); 1371 e.constant = NumToString(hashed_value); 1372 break; 1373 } 1374 case BASE_TYPE_LONG: { 1375 auto hash = FindHashFunction64(hash_name->constant.c_str()); 1376 int64_t hashed_value = static_cast<int64_t>(hash(attribute_.c_str())); 1377 e.constant = NumToString(hashed_value); 1378 break; 1379 } 1380 case BASE_TYPE_ULONG: { 1381 auto hash = FindHashFunction64(hash_name->constant.c_str()); 1382 uint64_t hashed_value = hash(attribute_.c_str()); 1383 e.constant = NumToString(hashed_value); 1384 break; 1385 } 1386 default: FLATBUFFERS_ASSERT(0); 1387 } 1388 NEXT(); 1389 return NoError(); 1390 } 1391 1392 CheckedError Parser::TokenError() { 1393 return Error("cannot parse value starting with: " + TokenToStringId(token_)); 1394 } 1395 1396 CheckedError Parser::ParseSingleValue(const std::string *name, Value &e, 1397 bool check_now) { 1398 // First see if this could be a conversion function: 1399 if (token_ == kTokenIdentifier && *cursor_ == '(') { 1400 // todo: Extract processing of conversion functions to ParseFunction. 1401 const auto functionname = attribute_; 1402 if (!IsFloat(e.type.base_type)) { 1403 return Error(functionname + ": type of argument mismatch, expecting: " + 1404 kTypeNames[BASE_TYPE_DOUBLE] + 1405 ", found: " + kTypeNames[e.type.base_type] + 1406 ", name: " + (name ? *name : "") + ", value: " + e.constant); 1407 } 1408 NEXT(); 1409 EXPECT('('); 1410 ECHECK(Recurse([&]() { return ParseSingleValue(name, e, false); })); 1411 EXPECT(')'); 1412 // calculate with double precision 1413 double x, y = 0.0; 1414 ECHECK(atot(e.constant.c_str(), *this, &x)); 1415 auto func_match = false; 1416 // clang-format off 1417 #define FLATBUFFERS_FN_DOUBLE(name, op) \ 1418 if (!func_match && functionname == name) { y = op; func_match = true; } 1419 FLATBUFFERS_FN_DOUBLE("deg", x / kPi * 180); 1420 FLATBUFFERS_FN_DOUBLE("rad", x * kPi / 180); 1421 FLATBUFFERS_FN_DOUBLE("sin", sin(x)); 1422 FLATBUFFERS_FN_DOUBLE("cos", cos(x)); 1423 FLATBUFFERS_FN_DOUBLE("tan", tan(x)); 1424 FLATBUFFERS_FN_DOUBLE("asin", asin(x)); 1425 FLATBUFFERS_FN_DOUBLE("acos", acos(x)); 1426 FLATBUFFERS_FN_DOUBLE("atan", atan(x)); 1427 // TODO(wvo): add more useful conversion functions here. 1428 #undef FLATBUFFERS_FN_DOUBLE 1429 // clang-format on 1430 if (true != func_match) { 1431 return Error(std::string("Unknown conversion function: ") + functionname + 1432 ", field name: " + (name ? *name : "") + 1433 ", value: " + e.constant); 1434 } 1435 e.constant = NumToString(y); 1436 return NoError(); 1437 } 1438 1439 auto match = false; 1440 // clang-format off 1441 #define TRY_ECHECK(force, dtoken, check, req) \ 1442 if (!match && ((check) || IsConstTrue(force))) \ 1443 ECHECK(TryTypedValue(name, dtoken, check, e, req, &match)) 1444 // clang-format on 1445 1446 if (token_ == kTokenStringConstant || token_ == kTokenIdentifier) { 1447 const auto kTokenStringOrIdent = token_; 1448 // The string type is a most probable type, check it first. 1449 TRY_ECHECK(false, kTokenStringConstant, 1450 e.type.base_type == BASE_TYPE_STRING, BASE_TYPE_STRING); 1451 1452 // avoid escaped and non-ascii in the string 1453 if ((token_ == kTokenStringConstant) && IsScalar(e.type.base_type) && 1454 !attr_is_trivial_ascii_string_) { 1455 return Error( 1456 std::string("type mismatch or invalid value, an initializer of " 1457 "non-string field must be trivial ASCII string: type: ") + 1458 kTypeNames[e.type.base_type] + ", name: " + (name ? *name : "") + 1459 ", value: " + attribute_); 1460 } 1461 1462 // A boolean as true/false. Boolean as Integer check below. 1463 if (!match && IsBool(e.type.base_type)) { 1464 auto is_true = attribute_ == "true"; 1465 if (is_true || attribute_ == "false") { 1466 attribute_ = is_true ? "1" : "0"; 1467 // accepts both kTokenStringConstant and kTokenIdentifier 1468 TRY_ECHECK(false, kTokenStringOrIdent, IsBool(e.type.base_type), 1469 BASE_TYPE_BOOL); 1470 } 1471 } 1472 // Check if this could be a string/identifier enum value. 1473 // Enum can have only true integer base type. 1474 if (!match && IsInteger(e.type.base_type) && !IsBool(e.type.base_type) && 1475 IsIdentifierStart(*attribute_.c_str())) { 1476 int64_t val; 1477 ECHECK(ParseEnumFromString(e.type, &val)); 1478 e.constant = NumToString(val); 1479 NEXT(); 1480 match = true; 1481 } 1482 // float/integer number in string 1483 if ((token_ == kTokenStringConstant) && IsScalar(e.type.base_type)) { 1484 // remove trailing whitespaces from attribute_ 1485 auto last = attribute_.find_last_not_of(' '); 1486 if (std::string::npos != last) // has non-whitespace 1487 attribute_.resize(last + 1); 1488 } 1489 // Float numbers or nan, inf, pi, etc. 1490 TRY_ECHECK(false, kTokenStringOrIdent, IsFloat(e.type.base_type), 1491 BASE_TYPE_FLOAT); 1492 // An integer constant in string. 1493 TRY_ECHECK(false, kTokenStringOrIdent, IsInteger(e.type.base_type), 1494 BASE_TYPE_INT); 1495 // Unknown tokens will be interpreted as string type. 1496 TRY_ECHECK(true, kTokenStringConstant, e.type.base_type == BASE_TYPE_STRING, 1497 BASE_TYPE_STRING); 1498 } else { 1499 // Try a float number. 1500 TRY_ECHECK(false, kTokenFloatConstant, IsFloat(e.type.base_type), 1501 BASE_TYPE_FLOAT); 1502 // Integer token can init any scalar (integer of float). 1503 TRY_ECHECK(true, kTokenIntegerConstant, IsScalar(e.type.base_type), 1504 BASE_TYPE_INT); 1505 } 1506 #undef TRY_ECHECK 1507 1508 if (!match) return TokenError(); 1509 1510 // The check_now flag must be true when parse a fbs-schema. 1511 // This flag forces to check default scalar values or metadata of field. 1512 // For JSON parser the flag should be false. 1513 // If it is set for JSON each value will be checked twice (see ParseTable). 1514 if (check_now && IsScalar(e.type.base_type)) { 1515 // "re-pack" an integer scalar to remove any ambiguities like leading zeros 1516 // which can be treated as octal-literal (idl_gen_cpp/GenDefaultConstant). 1517 const auto repack = IsInteger(e.type.base_type); 1518 switch (e.type.base_type) { 1519 // clang-format off 1520 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ 1521 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ 1522 case BASE_TYPE_ ## ENUM: {\ 1523 CTYPE val; \ 1524 ECHECK(atot(e.constant.c_str(), *this, &val)); \ 1525 if(repack) e.constant = NumToString(val); \ 1526 break; } 1527 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD); 1528 #undef FLATBUFFERS_TD 1529 default: break; 1530 // clang-format on 1531 } 1532 } 1533 return NoError(); 1534 } 1535 1536 StructDef *Parser::LookupCreateStruct(const std::string &name, 1537 bool create_if_new, bool definition) { 1538 std::string qualified_name = current_namespace_->GetFullyQualifiedName(name); 1539 // See if it exists pre-declared by an unqualified use. 1540 auto struct_def = LookupStruct(name); 1541 if (struct_def && struct_def->predecl) { 1542 if (definition) { 1543 // Make sure it has the current namespace, and is registered under its 1544 // qualified name. 1545 struct_def->defined_namespace = current_namespace_; 1546 structs_.Move(name, qualified_name); 1547 } 1548 return struct_def; 1549 } 1550 // See if it exists pre-declared by an qualified use. 1551 struct_def = LookupStruct(qualified_name); 1552 if (struct_def && struct_def->predecl) { 1553 if (definition) { 1554 // Make sure it has the current namespace. 1555 struct_def->defined_namespace = current_namespace_; 1556 } 1557 return struct_def; 1558 } 1559 if (!definition) { 1560 // Search thru parent namespaces. 1561 for (size_t components = current_namespace_->components.size(); 1562 components && !struct_def; components--) { 1563 struct_def = LookupStruct( 1564 current_namespace_->GetFullyQualifiedName(name, components - 1)); 1565 } 1566 } 1567 if (!struct_def && create_if_new) { 1568 struct_def = new StructDef(); 1569 if (definition) { 1570 structs_.Add(qualified_name, struct_def); 1571 struct_def->name = name; 1572 struct_def->defined_namespace = current_namespace_; 1573 } else { 1574 // Not a definition. 1575 // Rather than failing, we create a "pre declared" StructDef, due to 1576 // circular references, and check for errors at the end of parsing. 1577 // It is defined in the current namespace, as the best guess what the 1578 // final namespace will be. 1579 structs_.Add(name, struct_def); 1580 struct_def->name = name; 1581 struct_def->defined_namespace = current_namespace_; 1582 struct_def->original_location.reset( 1583 new std::string(file_being_parsed_ + ":" + NumToString(line_))); 1584 } 1585 } 1586 return struct_def; 1587 } 1588 1589 CheckedError Parser::ParseEnum(bool is_union, EnumDef **dest) { 1590 std::vector<std::string> enum_comment = doc_comment_; 1591 NEXT(); 1592 std::string enum_name = attribute_; 1593 EXPECT(kTokenIdentifier); 1594 EnumDef *enum_def; 1595 ECHECK(StartEnum(enum_name, is_union, &enum_def)); 1596 enum_def->doc_comment = enum_comment; 1597 if (!is_union && !opts.proto_mode) { 1598 // Give specialized error message, since this type spec used to 1599 // be optional in the first FlatBuffers release. 1600 if (!Is(':')) { 1601 return Error( 1602 "must specify the underlying integer type for this" 1603 " enum (e.g. \': short\', which was the default)."); 1604 } else { 1605 NEXT(); 1606 } 1607 // Specify the integer type underlying this enum. 1608 ECHECK(ParseType(enum_def->underlying_type)); 1609 if (!IsInteger(enum_def->underlying_type.base_type) || 1610 IsBool(enum_def->underlying_type.base_type)) 1611 return Error("underlying enum type must be integral"); 1612 // Make this type refer back to the enum it was derived from. 1613 enum_def->underlying_type.enum_def = enum_def; 1614 } 1615 ECHECK(ParseMetaData(&enum_def->attributes)); 1616 EXPECT('{'); 1617 if (is_union) enum_def->vals.Add("NONE", new EnumVal("NONE", 0)); 1618 std::set<std::pair<BaseType, StructDef*>> union_types; 1619 for (;;) { 1620 if (opts.proto_mode && attribute_ == "option") { 1621 ECHECK(ParseProtoOption()); 1622 } else { 1623 auto value_name = attribute_; 1624 auto full_name = value_name; 1625 std::vector<std::string> value_comment = doc_comment_; 1626 EXPECT(kTokenIdentifier); 1627 if (is_union) { 1628 ECHECK(ParseNamespacing(&full_name, &value_name)); 1629 if (opts.union_value_namespacing) { 1630 // Since we can't namespace the actual enum identifiers, turn 1631 // namespace parts into part of the identifier. 1632 value_name = full_name; 1633 std::replace(value_name.begin(), value_name.end(), '.', '_'); 1634 } 1635 } 1636 auto prevsize = enum_def->vals.vec.size(); 1637 auto prevvalue = prevsize > 0 ? enum_def->vals.vec.back()->value : 0; 1638 auto &ev = *new EnumVal(value_name, 0); 1639 if (enum_def->vals.Add(value_name, &ev)) 1640 return Error("enum value already exists: " + value_name); 1641 ev.doc_comment = value_comment; 1642 if (is_union) { 1643 if (Is(':')) { 1644 NEXT(); 1645 ECHECK(ParseType(ev.union_type)); 1646 if (ev.union_type.base_type != BASE_TYPE_STRUCT && 1647 ev.union_type.base_type != BASE_TYPE_STRING) 1648 return Error("union value type may only be table/struct/string"); 1649 } else { 1650 ev.union_type = Type(BASE_TYPE_STRUCT, LookupCreateStruct(full_name)); 1651 } 1652 if (!enum_def->uses_multiple_type_instances) { 1653 auto union_type_key = std::make_pair(ev.union_type.base_type, ev.union_type.struct_def); 1654 if (union_types.count(union_type_key) > 0) { 1655 enum_def->uses_multiple_type_instances = true; 1656 } else { 1657 union_types.insert(union_type_key); 1658 } 1659 } 1660 } 1661 if (Is('=')) { 1662 NEXT(); 1663 ECHECK(atot(attribute_.c_str(), *this, &ev.value)); 1664 EXPECT(kTokenIntegerConstant); 1665 if (!opts.proto_mode && prevsize && 1666 enum_def->vals.vec[prevsize - 1]->value >= ev.value) 1667 return Error("enum values must be specified in ascending order"); 1668 } else if (prevsize == 0) { 1669 // already set to zero 1670 } else if (prevvalue != flatbuffers::numeric_limits<int64_t>::max()) { 1671 ev.value = prevvalue + 1; 1672 } else { 1673 return Error("enum value overflows"); 1674 } 1675 1676 // Check that value fits into the underlying type. 1677 switch (enum_def->underlying_type.base_type) { 1678 // clang-format off 1679 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \ 1680 PTYPE, RTYPE) \ 1681 case BASE_TYPE_##ENUM: { \ 1682 int64_t min_value = static_cast<int64_t>( \ 1683 flatbuffers::numeric_limits<CTYPE>::lowest()); \ 1684 int64_t max_value = static_cast<int64_t>( \ 1685 flatbuffers::numeric_limits<CTYPE>::max()); \ 1686 if (ev.value < min_value || ev.value > max_value) { \ 1687 return Error( \ 1688 "enum value does not fit [" + NumToString(min_value) + \ 1689 "; " + NumToString(max_value) + "]"); \ 1690 } \ 1691 break; \ 1692 } 1693 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD); 1694 #undef FLATBUFFERS_TD 1695 default: break; 1696 // clang-format on 1697 } 1698 1699 if (opts.proto_mode && Is('[')) { 1700 NEXT(); 1701 // ignore attributes on enums. 1702 while (token_ != ']') NEXT(); 1703 NEXT(); 1704 } 1705 } 1706 if (!Is(opts.proto_mode ? ';' : ',')) break; 1707 NEXT(); 1708 if (Is('}')) break; 1709 } 1710 EXPECT('}'); 1711 if (enum_def->attributes.Lookup("bit_flags")) { 1712 for (auto it = enum_def->vals.vec.begin(); it != enum_def->vals.vec.end(); 1713 ++it) { 1714 if (static_cast<size_t>((*it)->value) >= 1715 SizeOf(enum_def->underlying_type.base_type) * 8) 1716 return Error("bit flag out of range of underlying integral type"); 1717 (*it)->value = 1LL << (*it)->value; 1718 } 1719 } 1720 if (dest) *dest = enum_def; 1721 types_.Add(current_namespace_->GetFullyQualifiedName(enum_def->name), 1722 new Type(BASE_TYPE_UNION, nullptr, enum_def)); 1723 return NoError(); 1724 } 1725 1726 CheckedError Parser::StartStruct(const std::string &name, StructDef **dest) { 1727 auto &struct_def = *LookupCreateStruct(name, true, true); 1728 if (!struct_def.predecl) return Error("datatype already exists: " + name); 1729 struct_def.predecl = false; 1730 struct_def.name = name; 1731 struct_def.file = file_being_parsed_; 1732 // Move this struct to the back of the vector just in case it was predeclared, 1733 // to preserve declaration order. 1734 *std::remove(structs_.vec.begin(), structs_.vec.end(), &struct_def) = 1735 &struct_def; 1736 *dest = &struct_def; 1737 return NoError(); 1738 } 1739 1740 CheckedError Parser::CheckClash(std::vector<FieldDef *> &fields, 1741 StructDef *struct_def, const char *suffix, 1742 BaseType basetype) { 1743 auto len = strlen(suffix); 1744 for (auto it = fields.begin(); it != fields.end(); ++it) { 1745 auto &fname = (*it)->name; 1746 if (fname.length() > len && 1747 fname.compare(fname.length() - len, len, suffix) == 0 && 1748 (*it)->value.type.base_type != BASE_TYPE_UTYPE) { 1749 auto field = 1750 struct_def->fields.Lookup(fname.substr(0, fname.length() - len)); 1751 if (field && field->value.type.base_type == basetype) 1752 return Error("Field " + fname + 1753 " would clash with generated functions for field " + 1754 field->name); 1755 } 1756 } 1757 return NoError(); 1758 } 1759 1760 bool Parser::SupportsVectorOfUnions() const { 1761 return opts.lang_to_generate != 0 && 1762 (opts.lang_to_generate & ~(IDLOptions::kCpp | IDLOptions::kJs | 1763 IDLOptions::kTs | IDLOptions::kPhp | 1764 IDLOptions::kJava | IDLOptions::kCSharp)) == 0; 1765 } 1766 1767 Namespace *Parser::UniqueNamespace(Namespace *ns) { 1768 for (auto it = namespaces_.begin(); it != namespaces_.end(); ++it) { 1769 if (ns->components == (*it)->components) { 1770 delete ns; 1771 return *it; 1772 } 1773 } 1774 namespaces_.push_back(ns); 1775 return ns; 1776 } 1777 1778 std::string Parser::UnqualifiedName(std::string full_qualified_name) { 1779 Namespace *ns = new Namespace(); 1780 1781 std::size_t current, previous = 0; 1782 current = full_qualified_name.find('.'); 1783 while (current != std::string::npos) { 1784 ns->components.push_back( 1785 full_qualified_name.substr(previous, current - previous)); 1786 previous = current + 1; 1787 current = full_qualified_name.find('.', previous); 1788 } 1789 current_namespace_ = UniqueNamespace(ns); 1790 return full_qualified_name.substr(previous, current - previous); 1791 } 1792 1793 static bool compareFieldDefs(const FieldDef *a, const FieldDef *b) { 1794 auto a_id = atoi(a->attributes.Lookup("id")->constant.c_str()); 1795 auto b_id = atoi(b->attributes.Lookup("id")->constant.c_str()); 1796 return a_id < b_id; 1797 } 1798 1799 CheckedError Parser::ParseDecl() { 1800 std::vector<std::string> dc = doc_comment_; 1801 bool fixed = IsIdent("struct"); 1802 if (!fixed && !IsIdent("table")) return Error("declaration expected"); 1803 NEXT(); 1804 std::string name = attribute_; 1805 EXPECT(kTokenIdentifier); 1806 StructDef *struct_def; 1807 ECHECK(StartStruct(name, &struct_def)); 1808 struct_def->doc_comment = dc; 1809 struct_def->fixed = fixed; 1810 ECHECK(ParseMetaData(&struct_def->attributes)); 1811 struct_def->sortbysize = 1812 struct_def->attributes.Lookup("original_order") == nullptr && !fixed; 1813 EXPECT('{'); 1814 while (token_ != '}') ECHECK(ParseField(*struct_def)); 1815 auto force_align = struct_def->attributes.Lookup("force_align"); 1816 if (fixed) { 1817 if (force_align) { 1818 auto align = static_cast<size_t>(atoi(force_align->constant.c_str())); 1819 if (force_align->type.base_type != BASE_TYPE_INT || 1820 align < struct_def->minalign || align > FLATBUFFERS_MAX_ALIGNMENT || 1821 align & (align - 1)) 1822 return Error( 1823 "force_align must be a power of two integer ranging from the" 1824 "struct\'s natural alignment to " + 1825 NumToString(FLATBUFFERS_MAX_ALIGNMENT)); 1826 struct_def->minalign = align; 1827 } 1828 if (!struct_def->bytesize) return Error("size 0 structs not allowed"); 1829 } 1830 struct_def->PadLastField(struct_def->minalign); 1831 // Check if this is a table that has manual id assignments 1832 auto &fields = struct_def->fields.vec; 1833 if (!fixed && fields.size()) { 1834 size_t num_id_fields = 0; 1835 for (auto it = fields.begin(); it != fields.end(); ++it) { 1836 if ((*it)->attributes.Lookup("id")) num_id_fields++; 1837 } 1838 // If any fields have ids.. 1839 if (num_id_fields) { 1840 // Then all fields must have them. 1841 if (num_id_fields != fields.size()) 1842 return Error( 1843 "either all fields or no fields must have an 'id' attribute"); 1844 // Simply sort by id, then the fields are the same as if no ids had 1845 // been specified. 1846 std::sort(fields.begin(), fields.end(), compareFieldDefs); 1847 // Verify we have a contiguous set, and reassign vtable offsets. 1848 for (int i = 0; i < static_cast<int>(fields.size()); i++) { 1849 if (i != atoi(fields[i]->attributes.Lookup("id")->constant.c_str())) 1850 return Error("field id\'s must be consecutive from 0, id " + 1851 NumToString(i) + " missing or set twice"); 1852 fields[i]->value.offset = FieldIndexToOffset(static_cast<voffset_t>(i)); 1853 } 1854 } 1855 } 1856 1857 ECHECK( 1858 CheckClash(fields, struct_def, UnionTypeFieldSuffix(), BASE_TYPE_UNION)); 1859 ECHECK(CheckClash(fields, struct_def, "Type", BASE_TYPE_UNION)); 1860 ECHECK(CheckClash(fields, struct_def, "_length", BASE_TYPE_VECTOR)); 1861 ECHECK(CheckClash(fields, struct_def, "Length", BASE_TYPE_VECTOR)); 1862 ECHECK(CheckClash(fields, struct_def, "_byte_vector", BASE_TYPE_STRING)); 1863 ECHECK(CheckClash(fields, struct_def, "ByteVector", BASE_TYPE_STRING)); 1864 EXPECT('}'); 1865 types_.Add(current_namespace_->GetFullyQualifiedName(struct_def->name), 1866 new Type(BASE_TYPE_STRUCT, struct_def, nullptr)); 1867 return NoError(); 1868 } 1869 1870 CheckedError Parser::ParseService() { 1871 std::vector<std::string> service_comment = doc_comment_; 1872 NEXT(); 1873 auto service_name = attribute_; 1874 EXPECT(kTokenIdentifier); 1875 auto &service_def = *new ServiceDef(); 1876 service_def.name = service_name; 1877 service_def.file = file_being_parsed_; 1878 service_def.doc_comment = service_comment; 1879 service_def.defined_namespace = current_namespace_; 1880 if (services_.Add(current_namespace_->GetFullyQualifiedName(service_name), 1881 &service_def)) 1882 return Error("service already exists: " + service_name); 1883 ECHECK(ParseMetaData(&service_def.attributes)); 1884 EXPECT('{'); 1885 do { 1886 std::vector<std::string> doc_comment = doc_comment_; 1887 auto rpc_name = attribute_; 1888 EXPECT(kTokenIdentifier); 1889 EXPECT('('); 1890 Type reqtype, resptype; 1891 ECHECK(ParseTypeIdent(reqtype)); 1892 EXPECT(')'); 1893 EXPECT(':'); 1894 ECHECK(ParseTypeIdent(resptype)); 1895 if (reqtype.base_type != BASE_TYPE_STRUCT || reqtype.struct_def->fixed || 1896 resptype.base_type != BASE_TYPE_STRUCT || resptype.struct_def->fixed) 1897 return Error("rpc request and response types must be tables"); 1898 auto &rpc = *new RPCCall(); 1899 rpc.name = rpc_name; 1900 rpc.request = reqtype.struct_def; 1901 rpc.response = resptype.struct_def; 1902 rpc.doc_comment = doc_comment; 1903 if (service_def.calls.Add(rpc_name, &rpc)) 1904 return Error("rpc already exists: " + rpc_name); 1905 ECHECK(ParseMetaData(&rpc.attributes)); 1906 EXPECT(';'); 1907 } while (token_ != '}'); 1908 NEXT(); 1909 return NoError(); 1910 } 1911 1912 bool Parser::SetRootType(const char *name) { 1913 root_struct_def_ = LookupStruct(name); 1914 if (!root_struct_def_) 1915 root_struct_def_ = 1916 LookupStruct(current_namespace_->GetFullyQualifiedName(name)); 1917 return root_struct_def_ != nullptr; 1918 } 1919 1920 void Parser::MarkGenerated() { 1921 // This function marks all existing definitions as having already 1922 // been generated, which signals no code for included files should be 1923 // generated. 1924 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) { 1925 (*it)->generated = true; 1926 } 1927 for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) { 1928 if (!(*it)->predecl) { (*it)->generated = true; } 1929 } 1930 for (auto it = services_.vec.begin(); it != services_.vec.end(); ++it) { 1931 (*it)->generated = true; 1932 } 1933 } 1934 1935 CheckedError Parser::ParseNamespace() { 1936 NEXT(); 1937 auto ns = new Namespace(); 1938 namespaces_.push_back(ns); // Store it here to not leak upon error. 1939 if (token_ != ';') { 1940 for (;;) { 1941 ns->components.push_back(attribute_); 1942 EXPECT(kTokenIdentifier); 1943 if (Is('.')) NEXT() else break; 1944 } 1945 } 1946 namespaces_.pop_back(); 1947 current_namespace_ = UniqueNamespace(ns); 1948 EXPECT(';'); 1949 return NoError(); 1950 } 1951 1952 static bool compareEnumVals(const EnumVal *a, const EnumVal *b) { 1953 return a->value < b->value; 1954 } 1955 1956 // Best effort parsing of .proto declarations, with the aim to turn them 1957 // in the closest corresponding FlatBuffer equivalent. 1958 // We parse everything as identifiers instead of keywords, since we don't 1959 // want protobuf keywords to become invalid identifiers in FlatBuffers. 1960 CheckedError Parser::ParseProtoDecl() { 1961 bool isextend = IsIdent("extend"); 1962 if (IsIdent("package")) { 1963 // These are identical in syntax to FlatBuffer's namespace decl. 1964 ECHECK(ParseNamespace()); 1965 } else if (IsIdent("message") || isextend) { 1966 std::vector<std::string> struct_comment = doc_comment_; 1967 NEXT(); 1968 StructDef *struct_def = nullptr; 1969 Namespace *parent_namespace = nullptr; 1970 if (isextend) { 1971 if (Is('.')) NEXT(); // qualified names may start with a . ? 1972 auto id = attribute_; 1973 EXPECT(kTokenIdentifier); 1974 ECHECK(ParseNamespacing(&id, nullptr)); 1975 struct_def = LookupCreateStruct(id, false); 1976 if (!struct_def) 1977 return Error("cannot extend unknown message type: " + id); 1978 } else { 1979 std::string name = attribute_; 1980 EXPECT(kTokenIdentifier); 1981 ECHECK(StartStruct(name, &struct_def)); 1982 // Since message definitions can be nested, we create a new namespace. 1983 auto ns = new Namespace(); 1984 // Copy of current namespace. 1985 *ns = *current_namespace_; 1986 // But with current message name. 1987 ns->components.push_back(name); 1988 ns->from_table++; 1989 parent_namespace = current_namespace_; 1990 current_namespace_ = UniqueNamespace(ns); 1991 } 1992 struct_def->doc_comment = struct_comment; 1993 ECHECK(ParseProtoFields(struct_def, isextend, false)); 1994 if (!isextend) { current_namespace_ = parent_namespace; } 1995 if (Is(';')) NEXT(); 1996 } else if (IsIdent("enum")) { 1997 // These are almost the same, just with different terminator: 1998 EnumDef *enum_def; 1999 ECHECK(ParseEnum(false, &enum_def)); 2000 if (Is(';')) NEXT(); 2001 // Protobuf allows them to be specified in any order, so sort afterwards. 2002 auto &v = enum_def->vals.vec; 2003 std::sort(v.begin(), v.end(), compareEnumVals); 2004 2005 // Temp: remove any duplicates, as .fbs files can't handle them. 2006 for (auto it = v.begin(); it != v.end();) { 2007 if (it != v.begin() && it[0]->value == it[-1]->value) 2008 it = v.erase(it); 2009 else 2010 ++it; 2011 } 2012 } else if (IsIdent("syntax")) { // Skip these. 2013 NEXT(); 2014 EXPECT('='); 2015 EXPECT(kTokenStringConstant); 2016 EXPECT(';'); 2017 } else if (IsIdent("option")) { // Skip these. 2018 ECHECK(ParseProtoOption()); 2019 EXPECT(';'); 2020 } else if (IsIdent("service")) { // Skip these. 2021 NEXT(); 2022 EXPECT(kTokenIdentifier); 2023 ECHECK(ParseProtoCurliesOrIdent()); 2024 } else { 2025 return Error("don\'t know how to parse .proto declaration starting with " + 2026 TokenToStringId(token_)); 2027 } 2028 return NoError(); 2029 } 2030 2031 CheckedError Parser::StartEnum(const std::string &enum_name, bool is_union, 2032 EnumDef **dest) { 2033 auto &enum_def = *new EnumDef(); 2034 enum_def.name = enum_name; 2035 enum_def.file = file_being_parsed_; 2036 enum_def.doc_comment = doc_comment_; 2037 enum_def.is_union = is_union; 2038 enum_def.defined_namespace = current_namespace_; 2039 if (enums_.Add(current_namespace_->GetFullyQualifiedName(enum_name), 2040 &enum_def)) 2041 return Error("enum already exists: " + enum_name); 2042 enum_def.underlying_type.base_type = is_union ? BASE_TYPE_UTYPE 2043 : BASE_TYPE_INT; 2044 enum_def.underlying_type.enum_def = &enum_def; 2045 if (dest) *dest = &enum_def; 2046 return NoError(); 2047 } 2048 2049 CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend, 2050 bool inside_oneof) { 2051 EXPECT('{'); 2052 while (token_ != '}') { 2053 if (IsIdent("message") || IsIdent("extend") || IsIdent("enum")) { 2054 // Nested declarations. 2055 ECHECK(ParseProtoDecl()); 2056 } else if (IsIdent("extensions")) { // Skip these. 2057 NEXT(); 2058 EXPECT(kTokenIntegerConstant); 2059 if (Is(kTokenIdentifier)) { 2060 NEXT(); // to 2061 NEXT(); // num 2062 } 2063 EXPECT(';'); 2064 } else if (IsIdent("option")) { // Skip these. 2065 ECHECK(ParseProtoOption()); 2066 EXPECT(';'); 2067 } else if (IsIdent("reserved")) { // Skip these. 2068 NEXT(); 2069 while (!Is(';')) { NEXT(); } // A variety of formats, just skip. 2070 NEXT(); 2071 } else { 2072 std::vector<std::string> field_comment = doc_comment_; 2073 // Parse the qualifier. 2074 bool required = false; 2075 bool repeated = false; 2076 bool oneof = false; 2077 if (!inside_oneof) { 2078 if (IsIdent("optional")) { 2079 // This is the default. 2080 NEXT(); 2081 } else if (IsIdent("required")) { 2082 required = true; 2083 NEXT(); 2084 } else if (IsIdent("repeated")) { 2085 repeated = true; 2086 NEXT(); 2087 } else if (IsIdent("oneof")) { 2088 oneof = true; 2089 NEXT(); 2090 } else { 2091 // can't error, proto3 allows decls without any of the above. 2092 } 2093 } 2094 StructDef *anonymous_struct = nullptr; 2095 EnumDef *oneof_union = nullptr; 2096 Type type; 2097 if (IsIdent("group") || oneof) { 2098 if (!oneof) NEXT(); 2099 if (oneof && opts.proto_oneof_union) { 2100 auto name = MakeCamel(attribute_, true) + "Union"; 2101 ECHECK(StartEnum(name, true, &oneof_union)); 2102 type = Type(BASE_TYPE_UNION, nullptr, oneof_union); 2103 } else { 2104 auto name = "Anonymous" + NumToString(anonymous_counter++); 2105 ECHECK(StartStruct(name, &anonymous_struct)); 2106 type = Type(BASE_TYPE_STRUCT, anonymous_struct); 2107 } 2108 } else { 2109 ECHECK(ParseTypeFromProtoType(&type)); 2110 } 2111 // Repeated elements get mapped to a vector. 2112 if (repeated) { 2113 type.element = type.base_type; 2114 type.base_type = BASE_TYPE_VECTOR; 2115 if (type.element == BASE_TYPE_VECTOR) { 2116 // We have a vector or vectors, which FlatBuffers doesn't support. 2117 // For now make it a vector of string (since the source is likely 2118 // "repeated bytes"). 2119 // TODO(wvo): A better solution would be to wrap this in a table. 2120 type.element = BASE_TYPE_STRING; 2121 } 2122 } 2123 std::string name = attribute_; 2124 EXPECT(kTokenIdentifier); 2125 if (!oneof) { 2126 // Parse the field id. Since we're just translating schemas, not 2127 // any kind of binary compatibility, we can safely ignore these, and 2128 // assign our own. 2129 EXPECT('='); 2130 EXPECT(kTokenIntegerConstant); 2131 } 2132 FieldDef *field = nullptr; 2133 if (isextend) { 2134 // We allow a field to be re-defined when extending. 2135 // TODO: are there situations where that is problematic? 2136 field = struct_def->fields.Lookup(name); 2137 } 2138 if (!field) ECHECK(AddField(*struct_def, name, type, &field)); 2139 field->doc_comment = field_comment; 2140 if (!IsScalar(type.base_type)) field->required = required; 2141 // See if there's a default specified. 2142 if (Is('[')) { 2143 NEXT(); 2144 for (;;) { 2145 auto key = attribute_; 2146 ECHECK(ParseProtoKey()); 2147 EXPECT('='); 2148 auto val = attribute_; 2149 ECHECK(ParseProtoCurliesOrIdent()); 2150 if (key == "default") { 2151 // Temp: skip non-numeric defaults (enums). 2152 auto numeric = strpbrk(val.c_str(), "0123456789-+."); 2153 if (IsScalar(type.base_type) && numeric == val.c_str()) 2154 field->value.constant = val; 2155 } else if (key == "deprecated") { 2156 field->deprecated = val == "true"; 2157 } 2158 if (!Is(',')) break; 2159 NEXT(); 2160 } 2161 EXPECT(']'); 2162 } 2163 if (anonymous_struct) { 2164 ECHECK(ParseProtoFields(anonymous_struct, false, oneof)); 2165 if (Is(';')) NEXT(); 2166 } else if (oneof_union) { 2167 // Parse into a temporary StructDef, then transfer fields into an 2168 // EnumDef describing the oneof as a union. 2169 StructDef oneof_struct; 2170 ECHECK(ParseProtoFields(&oneof_struct, false, oneof)); 2171 if (Is(';')) NEXT(); 2172 for (auto field_it = oneof_struct.fields.vec.begin(); 2173 field_it != oneof_struct.fields.vec.end(); ++field_it) { 2174 const auto &oneof_field = **field_it; 2175 const auto &oneof_type = oneof_field.value.type; 2176 if (oneof_type.base_type != BASE_TYPE_STRUCT || 2177 !oneof_type.struct_def || oneof_type.struct_def->fixed) 2178 return Error("oneof '" + name + 2179 "' cannot be mapped to a union because member '" + 2180 oneof_field.name + "' is not a table type."); 2181 auto enum_val = new EnumVal(oneof_type.struct_def->name, 2182 oneof_union->vals.vec.size()); 2183 enum_val->union_type = oneof_type; 2184 enum_val->doc_comment = oneof_field.doc_comment; 2185 oneof_union->vals.Add(oneof_field.name, enum_val); 2186 } 2187 } else { 2188 EXPECT(';'); 2189 } 2190 } 2191 } 2192 NEXT(); 2193 return NoError(); 2194 } 2195 2196 CheckedError Parser::ParseProtoKey() { 2197 if (token_ == '(') { 2198 NEXT(); 2199 // Skip "(a.b)" style custom attributes. 2200 while (token_ == '.' || token_ == kTokenIdentifier) NEXT(); 2201 EXPECT(')'); 2202 while (Is('.')) { 2203 NEXT(); 2204 EXPECT(kTokenIdentifier); 2205 } 2206 } else { 2207 EXPECT(kTokenIdentifier); 2208 } 2209 return NoError(); 2210 } 2211 2212 CheckedError Parser::ParseProtoCurliesOrIdent() { 2213 if (Is('{')) { 2214 NEXT(); 2215 for (int nesting = 1; nesting;) { 2216 if (token_ == '{') 2217 nesting++; 2218 else if (token_ == '}') 2219 nesting--; 2220 NEXT(); 2221 } 2222 } else { 2223 NEXT(); // Any single token. 2224 } 2225 return NoError(); 2226 } 2227 2228 CheckedError Parser::ParseProtoOption() { 2229 NEXT(); 2230 ECHECK(ParseProtoKey()); 2231 EXPECT('='); 2232 ECHECK(ParseProtoCurliesOrIdent()); 2233 return NoError(); 2234 } 2235 2236 // Parse a protobuf type, and map it to the corresponding FlatBuffer one. 2237 CheckedError Parser::ParseTypeFromProtoType(Type *type) { 2238 struct type_lookup { 2239 const char *proto_type; 2240 BaseType fb_type, element; 2241 }; 2242 static type_lookup lookup[] = { 2243 { "float", BASE_TYPE_FLOAT, BASE_TYPE_NONE }, 2244 { "double", BASE_TYPE_DOUBLE, BASE_TYPE_NONE }, 2245 { "int32", BASE_TYPE_INT, BASE_TYPE_NONE }, 2246 { "int64", BASE_TYPE_LONG, BASE_TYPE_NONE }, 2247 { "uint32", BASE_TYPE_UINT, BASE_TYPE_NONE }, 2248 { "uint64", BASE_TYPE_ULONG, BASE_TYPE_NONE }, 2249 { "sint32", BASE_TYPE_INT, BASE_TYPE_NONE }, 2250 { "sint64", BASE_TYPE_LONG, BASE_TYPE_NONE }, 2251 { "fixed32", BASE_TYPE_UINT, BASE_TYPE_NONE }, 2252 { "fixed64", BASE_TYPE_ULONG, BASE_TYPE_NONE }, 2253 { "sfixed32", BASE_TYPE_INT, BASE_TYPE_NONE }, 2254 { "sfixed64", BASE_TYPE_LONG, BASE_TYPE_NONE }, 2255 { "bool", BASE_TYPE_BOOL, BASE_TYPE_NONE }, 2256 { "string", BASE_TYPE_STRING, BASE_TYPE_NONE }, 2257 { "bytes", BASE_TYPE_VECTOR, BASE_TYPE_UCHAR }, 2258 { nullptr, BASE_TYPE_NONE, BASE_TYPE_NONE } 2259 }; 2260 for (auto tl = lookup; tl->proto_type; tl++) { 2261 if (attribute_ == tl->proto_type) { 2262 type->base_type = tl->fb_type; 2263 type->element = tl->element; 2264 NEXT(); 2265 return NoError(); 2266 } 2267 } 2268 if (Is('.')) NEXT(); // qualified names may start with a . ? 2269 ECHECK(ParseTypeIdent(*type)); 2270 return NoError(); 2271 } 2272 2273 CheckedError Parser::SkipAnyJsonValue() { 2274 switch (token_) { 2275 case '{': { 2276 size_t fieldn_outer = 0; 2277 return ParseTableDelimiters( 2278 fieldn_outer, nullptr, 2279 [&](const std::string &, size_t &fieldn, 2280 const StructDef *) -> CheckedError { 2281 ECHECK(Recurse([&]() { return SkipAnyJsonValue(); })); 2282 fieldn++; 2283 return NoError(); 2284 }); 2285 } 2286 case '[': { 2287 size_t count = 0; 2288 return ParseVectorDelimiters(count, [&](size_t &) -> CheckedError { 2289 return Recurse([&]() { return SkipAnyJsonValue(); }); 2290 }); 2291 } 2292 case kTokenStringConstant: 2293 case kTokenIntegerConstant: 2294 case kTokenFloatConstant: NEXT(); break; 2295 default: 2296 if (IsIdent("true") || IsIdent("false") || IsIdent("null")) { 2297 NEXT(); 2298 } else 2299 return TokenError(); 2300 } 2301 return NoError(); 2302 } 2303 2304 CheckedError Parser::ParseFlexBufferValue(flexbuffers::Builder *builder) { 2305 switch (token_) { 2306 case '{': { 2307 auto start = builder->StartMap(); 2308 size_t fieldn_outer = 0; 2309 auto err = 2310 ParseTableDelimiters(fieldn_outer, nullptr, 2311 [&](const std::string &name, size_t &fieldn, 2312 const StructDef *) -> CheckedError { 2313 builder->Key(name); 2314 ECHECK(ParseFlexBufferValue(builder)); 2315 fieldn++; 2316 return NoError(); 2317 }); 2318 ECHECK(err); 2319 builder->EndMap(start); 2320 break; 2321 } 2322 case '[': { 2323 auto start = builder->StartVector(); 2324 size_t count = 0; 2325 ECHECK(ParseVectorDelimiters(count, [&](size_t &) -> CheckedError { 2326 return ParseFlexBufferValue(builder); 2327 })); 2328 builder->EndVector(start, false, false); 2329 break; 2330 } 2331 case kTokenStringConstant: 2332 builder->String(attribute_); 2333 EXPECT(kTokenStringConstant); 2334 break; 2335 case kTokenIntegerConstant: 2336 builder->Int(StringToInt(attribute_.c_str())); 2337 EXPECT(kTokenIntegerConstant); 2338 break; 2339 case kTokenFloatConstant: 2340 builder->Double(strtod(attribute_.c_str(), nullptr)); 2341 EXPECT(kTokenFloatConstant); 2342 break; 2343 default: 2344 if (IsIdent("true")) { 2345 builder->Bool(true); 2346 NEXT(); 2347 } else if (IsIdent("false")) { 2348 builder->Bool(false); 2349 NEXT(); 2350 } else if (IsIdent("null")) { 2351 builder->Null(); 2352 NEXT(); 2353 } else 2354 return TokenError(); 2355 } 2356 return NoError(); 2357 } 2358 2359 bool Parser::ParseFlexBuffer(const char *source, const char *source_filename, 2360 flexbuffers::Builder *builder) { 2361 auto ok = !StartParseFile(source, source_filename).Check() && 2362 !ParseFlexBufferValue(builder).Check(); 2363 if (ok) builder->Finish(); 2364 return ok; 2365 } 2366 2367 bool Parser::Parse(const char *source, const char **include_paths, 2368 const char *source_filename) { 2369 FLATBUFFERS_ASSERT(0 == recurse_protection_counter); 2370 auto r = !ParseRoot(source, include_paths, source_filename).Check(); 2371 FLATBUFFERS_ASSERT(0 == recurse_protection_counter); 2372 return r; 2373 } 2374 2375 CheckedError Parser::StartParseFile(const char *source, 2376 const char *source_filename) { 2377 file_being_parsed_ = source_filename ? source_filename : ""; 2378 source_ = source; 2379 ResetState(source_); 2380 error_.clear(); 2381 ECHECK(SkipByteOrderMark()); 2382 NEXT(); 2383 if (Is(kTokenEof)) return Error("input file is empty"); 2384 return NoError(); 2385 } 2386 2387 CheckedError Parser::ParseRoot(const char *source, const char **include_paths, 2388 const char *source_filename) { 2389 ECHECK(DoParse(source, include_paths, source_filename, nullptr)); 2390 2391 // Check that all types were defined. 2392 for (auto it = structs_.vec.begin(); it != structs_.vec.end();) { 2393 auto &struct_def = **it; 2394 if (struct_def.predecl) { 2395 if (opts.proto_mode) { 2396 // Protos allow enums to be used before declaration, so check if that 2397 // is the case here. 2398 EnumDef *enum_def = nullptr; 2399 for (size_t components = 2400 struct_def.defined_namespace->components.size() + 1; 2401 components && !enum_def; components--) { 2402 auto qualified_name = 2403 struct_def.defined_namespace->GetFullyQualifiedName( 2404 struct_def.name, components - 1); 2405 enum_def = LookupEnum(qualified_name); 2406 } 2407 if (enum_def) { 2408 // This is pretty slow, but a simple solution for now. 2409 auto initial_count = struct_def.refcount; 2410 for (auto struct_it = structs_.vec.begin(); 2411 struct_it != structs_.vec.end(); ++struct_it) { 2412 auto &sd = **struct_it; 2413 for (auto field_it = sd.fields.vec.begin(); 2414 field_it != sd.fields.vec.end(); ++field_it) { 2415 auto &field = **field_it; 2416 if (field.value.type.struct_def == &struct_def) { 2417 field.value.type.struct_def = nullptr; 2418 field.value.type.enum_def = enum_def; 2419 auto &bt = field.value.type.base_type == BASE_TYPE_VECTOR 2420 ? field.value.type.element 2421 : field.value.type.base_type; 2422 FLATBUFFERS_ASSERT(bt == BASE_TYPE_STRUCT); 2423 bt = enum_def->underlying_type.base_type; 2424 struct_def.refcount--; 2425 enum_def->refcount++; 2426 } 2427 } 2428 } 2429 if (struct_def.refcount) 2430 return Error("internal: " + NumToString(struct_def.refcount) + "/" + 2431 NumToString(initial_count) + 2432 " use(s) of pre-declaration enum not accounted for: " + 2433 enum_def->name); 2434 structs_.dict.erase(structs_.dict.find(struct_def.name)); 2435 it = structs_.vec.erase(it); 2436 delete &struct_def; 2437 continue; // Skip error. 2438 } 2439 } 2440 auto err = "type referenced but not defined (check namespace): " + 2441 struct_def.name; 2442 if (struct_def.original_location) 2443 err += ", originally at: " + *struct_def.original_location; 2444 return Error(err); 2445 } 2446 ++it; 2447 } 2448 2449 // This check has to happen here and not earlier, because only now do we 2450 // know for sure what the type of these are. 2451 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) { 2452 auto &enum_def = **it; 2453 if (enum_def.is_union) { 2454 for (auto val_it = enum_def.vals.vec.begin(); 2455 val_it != enum_def.vals.vec.end(); ++val_it) { 2456 auto &val = **val_it; 2457 if (!SupportsVectorOfUnions() && val.union_type.struct_def && 2458 val.union_type.struct_def->fixed) 2459 return Error( 2460 "only tables can be union elements in the generated language: " + 2461 val.name); 2462 } 2463 } 2464 } 2465 return NoError(); 2466 } 2467 2468 CheckedError Parser::DoParse(const char *source, const char **include_paths, 2469 const char *source_filename, 2470 const char *include_filename) { 2471 if (source_filename) { 2472 if (included_files_.find(source_filename) == included_files_.end()) { 2473 included_files_[source_filename] = 2474 include_filename ? include_filename : ""; 2475 files_included_per_file_[source_filename] = std::set<std::string>(); 2476 } else { 2477 return NoError(); 2478 } 2479 } 2480 if (!include_paths) { 2481 static const char *current_directory[] = { "", nullptr }; 2482 include_paths = current_directory; 2483 } 2484 field_stack_.clear(); 2485 builder_.Clear(); 2486 // Start with a blank namespace just in case this file doesn't have one. 2487 current_namespace_ = empty_namespace_; 2488 2489 ECHECK(StartParseFile(source, source_filename)); 2490 2491 // Includes must come before type declarations: 2492 for (;;) { 2493 // Parse pre-include proto statements if any: 2494 if (opts.proto_mode && (attribute_ == "option" || attribute_ == "syntax" || 2495 attribute_ == "package")) { 2496 ECHECK(ParseProtoDecl()); 2497 } else if (IsIdent("native_include")) { 2498 NEXT(); 2499 vector_emplace_back(&native_included_files_, attribute_); 2500 EXPECT(kTokenStringConstant); 2501 EXPECT(';'); 2502 } else if (IsIdent("include") || (opts.proto_mode && IsIdent("import"))) { 2503 NEXT(); 2504 if (opts.proto_mode && attribute_ == "public") NEXT(); 2505 auto name = flatbuffers::PosixPath(attribute_.c_str()); 2506 EXPECT(kTokenStringConstant); 2507 // Look for the file in include_paths. 2508 std::string filepath; 2509 for (auto paths = include_paths; paths && *paths; paths++) { 2510 filepath = flatbuffers::ConCatPathFileName(*paths, name); 2511 if (FileExists(filepath.c_str())) break; 2512 } 2513 if (filepath.empty()) 2514 return Error("unable to locate include file: " + name); 2515 if (source_filename) 2516 files_included_per_file_[source_filename].insert(filepath); 2517 if (included_files_.find(filepath) == included_files_.end()) { 2518 // We found an include file that we have not parsed yet. 2519 // Load it and parse it. 2520 std::string contents; 2521 if (!LoadFile(filepath.c_str(), true, &contents)) 2522 return Error("unable to load include file: " + name); 2523 ECHECK(DoParse(contents.c_str(), include_paths, filepath.c_str(), 2524 name.c_str())); 2525 // We generally do not want to output code for any included files: 2526 if (!opts.generate_all) MarkGenerated(); 2527 // Reset these just in case the included file had them, and the 2528 // parent doesn't. 2529 root_struct_def_ = nullptr; 2530 file_identifier_.clear(); 2531 file_extension_.clear(); 2532 // This is the easiest way to continue this file after an include: 2533 // instead of saving and restoring all the state, we simply start the 2534 // file anew. This will cause it to encounter the same include 2535 // statement again, but this time it will skip it, because it was 2536 // entered into included_files_. 2537 // This is recursive, but only go as deep as the number of include 2538 // statements. 2539 if (source_filename) { 2540 included_files_.erase(source_filename); 2541 } 2542 return DoParse(source, include_paths, source_filename, 2543 include_filename); 2544 } 2545 EXPECT(';'); 2546 } else { 2547 break; 2548 } 2549 } 2550 // Now parse all other kinds of declarations: 2551 while (token_ != kTokenEof) { 2552 if (opts.proto_mode) { 2553 ECHECK(ParseProtoDecl()); 2554 } else if (IsIdent("namespace")) { 2555 ECHECK(ParseNamespace()); 2556 } else if (token_ == '{') { 2557 if (!root_struct_def_) 2558 return Error("no root type set to parse json with"); 2559 if (builder_.GetSize()) { 2560 return Error("cannot have more than one json object in a file"); 2561 } 2562 uoffset_t toff; 2563 ECHECK(ParseTable(*root_struct_def_, nullptr, &toff)); 2564 if (opts.size_prefixed) { 2565 builder_.FinishSizePrefixed(Offset<Table>(toff), file_identifier_.length() 2566 ? file_identifier_.c_str() 2567 : nullptr); 2568 } else { 2569 builder_.Finish(Offset<Table>(toff), file_identifier_.length() 2570 ? file_identifier_.c_str() 2571 : nullptr); 2572 } 2573 // Check that JSON file doesn't contain more objects or IDL directives. 2574 // Comments after JSON are allowed. 2575 EXPECT(kTokenEof); 2576 } else if (IsIdent("enum")) { 2577 ECHECK(ParseEnum(false, nullptr)); 2578 } else if (IsIdent("union")) { 2579 ECHECK(ParseEnum(true, nullptr)); 2580 } else if (IsIdent("root_type")) { 2581 NEXT(); 2582 auto root_type = attribute_; 2583 EXPECT(kTokenIdentifier); 2584 ECHECK(ParseNamespacing(&root_type, nullptr)); 2585 if (opts.root_type.empty()) { 2586 if (!SetRootType(root_type.c_str())) 2587 return Error("unknown root type: " + root_type); 2588 if (root_struct_def_->fixed) 2589 return Error("root type must be a table"); 2590 } 2591 EXPECT(';'); 2592 } else if (IsIdent("file_identifier")) { 2593 NEXT(); 2594 file_identifier_ = attribute_; 2595 EXPECT(kTokenStringConstant); 2596 if (file_identifier_.length() != FlatBufferBuilder::kFileIdentifierLength) 2597 return Error("file_identifier must be exactly " + 2598 NumToString(FlatBufferBuilder::kFileIdentifierLength) + 2599 " characters"); 2600 EXPECT(';'); 2601 } else if (IsIdent("file_extension")) { 2602 NEXT(); 2603 file_extension_ = attribute_; 2604 EXPECT(kTokenStringConstant); 2605 EXPECT(';'); 2606 } else if (IsIdent("include")) { 2607 return Error("includes must come before declarations"); 2608 } else if (IsIdent("attribute")) { 2609 NEXT(); 2610 auto name = attribute_; 2611 if (Is(kTokenIdentifier)) { 2612 NEXT(); 2613 } else { 2614 EXPECT(kTokenStringConstant); 2615 } 2616 EXPECT(';'); 2617 known_attributes_[name] = false; 2618 } else if (IsIdent("rpc_service")) { 2619 ECHECK(ParseService()); 2620 } else { 2621 ECHECK(ParseDecl()); 2622 } 2623 } 2624 return NoError(); 2625 } 2626 2627 std::set<std::string> Parser::GetIncludedFilesRecursive( 2628 const std::string &file_name) const { 2629 std::set<std::string> included_files; 2630 std::list<std::string> to_process; 2631 2632 if (file_name.empty()) return included_files; 2633 to_process.push_back(file_name); 2634 2635 while (!to_process.empty()) { 2636 std::string current = to_process.front(); 2637 to_process.pop_front(); 2638 included_files.insert(current); 2639 2640 // Workaround the lack of const accessor in C++98 maps. 2641 auto &new_files = 2642 (*const_cast<std::map<std::string, std::set<std::string>> *>( 2643 &files_included_per_file_))[current]; 2644 for (auto it = new_files.begin(); it != new_files.end(); ++it) { 2645 if (included_files.find(*it) == included_files.end()) 2646 to_process.push_back(*it); 2647 } 2648 } 2649 2650 return included_files; 2651 } 2652 2653 // Schema serialization functionality: 2654 2655 template<typename T> bool compareName(const T *a, const T *b) { 2656 return a->defined_namespace->GetFullyQualifiedName(a->name) < 2657 b->defined_namespace->GetFullyQualifiedName(b->name); 2658 } 2659 2660 template<typename T> void AssignIndices(const std::vector<T *> &defvec) { 2661 // Pre-sort these vectors, such that we can set the correct indices for them. 2662 auto vec = defvec; 2663 std::sort(vec.begin(), vec.end(), compareName<T>); 2664 for (int i = 0; i < static_cast<int>(vec.size()); i++) vec[i]->index = i; 2665 } 2666 2667 void Parser::Serialize() { 2668 builder_.Clear(); 2669 AssignIndices(structs_.vec); 2670 AssignIndices(enums_.vec); 2671 std::vector<Offset<reflection::Object>> object_offsets; 2672 for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) { 2673 auto offset = (*it)->Serialize(&builder_, *this); 2674 object_offsets.push_back(offset); 2675 (*it)->serialized_location = offset.o; 2676 } 2677 std::vector<Offset<reflection::Enum>> enum_offsets; 2678 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) { 2679 auto offset = (*it)->Serialize(&builder_, *this); 2680 enum_offsets.push_back(offset); 2681 (*it)->serialized_location = offset.o; 2682 } 2683 std::vector<Offset<reflection::Service>> service_offsets; 2684 for (auto it = services_.vec.begin(); it != services_.vec.end(); ++it) { 2685 auto offset = (*it)->Serialize(&builder_, *this); 2686 service_offsets.push_back(offset); 2687 (*it)->serialized_location = offset.o; 2688 } 2689 auto objs__ = builder_.CreateVectorOfSortedTables(&object_offsets); 2690 auto enum__ = builder_.CreateVectorOfSortedTables(&enum_offsets); 2691 auto fiid__ = builder_.CreateString(file_identifier_); 2692 auto fext__ = builder_.CreateString(file_extension_); 2693 auto serv__ = builder_.CreateVectorOfSortedTables(&service_offsets); 2694 auto schema_offset = 2695 reflection::CreateSchema(builder_, objs__, enum__, fiid__, fext__, 2696 (root_struct_def_ ? root_struct_def_->serialized_location : 0), 2697 serv__); 2698 if (opts.size_prefixed) { 2699 builder_.FinishSizePrefixed(schema_offset, reflection::SchemaIdentifier()); 2700 } else { 2701 builder_.Finish(schema_offset, reflection::SchemaIdentifier()); 2702 } 2703 } 2704 2705 static Namespace *GetNamespace( 2706 const std::string &qualified_name, std::vector<Namespace *> &namespaces, 2707 std::map<std::string, Namespace *> &namespaces_index) { 2708 size_t dot = qualified_name.find_last_of('.'); 2709 std::string namespace_name = (dot != std::string::npos) 2710 ? std::string(qualified_name.c_str(), dot) 2711 : ""; 2712 Namespace *&ns = namespaces_index[namespace_name]; 2713 2714 if (!ns) { 2715 ns = new Namespace(); 2716 namespaces.push_back(ns); 2717 2718 size_t pos = 0; 2719 2720 for (;;) { 2721 dot = qualified_name.find('.', pos); 2722 if (dot == std::string::npos) { break; } 2723 ns->components.push_back(qualified_name.substr(pos, dot-pos)); 2724 pos = dot + 1; 2725 } 2726 } 2727 2728 return ns; 2729 } 2730 2731 Offset<reflection::Object> StructDef::Serialize(FlatBufferBuilder *builder, 2732 const Parser &parser) const { 2733 std::vector<Offset<reflection::Field>> field_offsets; 2734 for (auto it = fields.vec.begin(); it != fields.vec.end(); ++it) { 2735 field_offsets.push_back((*it)->Serialize( 2736 builder, static_cast<uint16_t>(it - fields.vec.begin()), parser)); 2737 } 2738 auto qualified_name = defined_namespace->GetFullyQualifiedName(name); 2739 auto name__ = builder->CreateString(qualified_name); 2740 auto flds__ = builder->CreateVectorOfSortedTables(&field_offsets); 2741 auto attr__ = SerializeAttributes(builder, parser); 2742 auto docs__ = parser.opts.binary_schema_comments 2743 ? builder->CreateVectorOfStrings(doc_comment) 2744 : 0; 2745 return reflection::CreateObject(*builder, name__, flds__, fixed, 2746 static_cast<int>(minalign), 2747 static_cast<int>(bytesize), 2748 attr__, docs__); 2749 } 2750 2751 bool StructDef::Deserialize(Parser &parser, const reflection::Object *object) { 2752 if (!DeserializeAttributes(parser, object->attributes())) 2753 return false; 2754 DeserializeDoc(doc_comment, object->documentation()); 2755 name = parser.UnqualifiedName(object->name()->str()); 2756 fixed = object->is_struct(); 2757 minalign = object->minalign(); 2758 predecl = false; 2759 sortbysize = attributes.Lookup("original_order") == nullptr && !fixed; 2760 std::vector<uoffset_t> indexes = 2761 std::vector<uoffset_t>(object->fields()->size()); 2762 for (uoffset_t i = 0; i < object->fields()->size(); i++) 2763 indexes[object->fields()->Get(i)->id()] = i; 2764 for (size_t i = 0; i < indexes.size(); i++) { 2765 auto field = object->fields()->Get(indexes[i]); 2766 auto field_def = new FieldDef(); 2767 if (!field_def->Deserialize(parser, field) || 2768 fields.Add(field_def->name, field_def)) { 2769 delete field_def; 2770 return false; 2771 } 2772 if (fixed) { 2773 // Recompute padding since that's currently not serialized. 2774 auto size = InlineSize(field_def->value.type); 2775 auto next_field = 2776 i + 1 < indexes.size() 2777 ? object->fields()->Get(indexes[i+1]) 2778 : nullptr; 2779 bytesize += size; 2780 field_def->padding = 2781 next_field ? (next_field->offset() - field_def->value.offset) - size 2782 : PaddingBytes(bytesize, minalign); 2783 bytesize += field_def->padding; 2784 } 2785 } 2786 FLATBUFFERS_ASSERT(static_cast<int>(bytesize) == object->bytesize()); 2787 return true; 2788 } 2789 2790 Offset<reflection::Field> FieldDef::Serialize(FlatBufferBuilder *builder, 2791 uint16_t id, 2792 const Parser &parser) const { 2793 auto name__ = builder->CreateString(name); 2794 auto type__ = value.type.Serialize(builder); 2795 auto attr__ = SerializeAttributes(builder, parser); 2796 auto docs__ = parser.opts.binary_schema_comments 2797 ? builder->CreateVectorOfStrings(doc_comment) 2798 : 0; 2799 return reflection::CreateField(*builder, name__, type__, id, value.offset, 2800 // Is uint64>max(int64) tested? 2801 IsInteger(value.type.base_type) ? StringToInt(value.constant.c_str()) : 0, 2802 // result may be platform-dependent if underlying is float (not double) 2803 IsFloat(value.type.base_type) ? strtod(value.constant.c_str(), nullptr) 2804 : 0.0, 2805 deprecated, required, key, attr__, docs__); 2806 // TODO: value.constant is almost always "0", we could save quite a bit of 2807 // space by sharing it. Same for common values of value.type. 2808 } 2809 2810 bool FieldDef::Deserialize(Parser &parser, const reflection::Field *field) { 2811 name = parser.UnqualifiedName(field->name()->str()); 2812 defined_namespace = parser.current_namespace_; 2813 if (!value.type.Deserialize(parser, field->type())) 2814 return false; 2815 value.offset = field->offset(); 2816 if (IsInteger(value.type.base_type)) { 2817 value.constant = NumToString(field->default_integer()); 2818 } else if (IsFloat(value.type.base_type)) { 2819 value.constant = FloatToString(field->default_real(), 16); 2820 size_t last_zero = value.constant.find_last_not_of('0'); 2821 if (last_zero != std::string::npos && last_zero != 0) { 2822 value.constant.erase(last_zero, std::string::npos); 2823 } 2824 } 2825 deprecated = field->deprecated(); 2826 required = field->required(); 2827 key = field->key(); 2828 if (!DeserializeAttributes(parser, field->attributes())) 2829 return false; 2830 // TODO: this should probably be handled by a separate attribute 2831 if (attributes.Lookup("flexbuffer")) { 2832 flexbuffer = true; 2833 parser.uses_flexbuffers_ = true; 2834 if (value.type.base_type != BASE_TYPE_VECTOR || 2835 value.type.element != BASE_TYPE_UCHAR) 2836 return false; 2837 } 2838 DeserializeDoc(doc_comment, field->documentation()); 2839 return true; 2840 } 2841 2842 Offset<reflection::RPCCall> RPCCall::Serialize(FlatBufferBuilder *builder, 2843 const Parser &parser) const { 2844 auto name__ = builder->CreateString(name); 2845 auto attr__ = SerializeAttributes(builder, parser); 2846 auto docs__ = parser.opts.binary_schema_comments 2847 ? builder->CreateVectorOfStrings(doc_comment) 2848 : 0; 2849 return reflection::CreateRPCCall(*builder, name__, 2850 request->serialized_location, 2851 response->serialized_location, 2852 attr__, docs__); 2853 } 2854 2855 bool RPCCall::Deserialize(Parser &parser, const reflection::RPCCall *call) { 2856 name = call->name()->str(); 2857 if (!DeserializeAttributes(parser, call->attributes())) 2858 return false; 2859 DeserializeDoc(doc_comment, call->documentation()); 2860 request = parser.structs_.Lookup(call->request()->name()->str()); 2861 response = parser.structs_.Lookup(call->response()->name()->str()); 2862 if (!request || !response) { return false; } 2863 return true; 2864 } 2865 2866 Offset<reflection::Service> ServiceDef::Serialize(FlatBufferBuilder *builder, 2867 const Parser &parser) const { 2868 std::vector<Offset<reflection::RPCCall>> servicecall_offsets; 2869 for (auto it = calls.vec.begin(); it != calls.vec.end(); ++it) { 2870 servicecall_offsets.push_back((*it)->Serialize(builder, parser)); 2871 } 2872 auto qualified_name = defined_namespace->GetFullyQualifiedName(name); 2873 auto name__ = builder->CreateString(qualified_name); 2874 auto call__ = builder->CreateVector(servicecall_offsets); 2875 auto attr__ = SerializeAttributes(builder, parser); 2876 auto docs__ = parser.opts.binary_schema_comments 2877 ? builder->CreateVectorOfStrings(doc_comment) 2878 : 0; 2879 return reflection::CreateService(*builder, name__, call__, attr__, docs__); 2880 } 2881 2882 bool ServiceDef::Deserialize(Parser &parser, 2883 const reflection::Service *service) { 2884 name = parser.UnqualifiedName(service->name()->str()); 2885 if (service->calls()) { 2886 for (uoffset_t i = 0; i < service->calls()->size(); ++i) { 2887 auto call = new RPCCall(); 2888 if (!call->Deserialize(parser, service->calls()->Get(i)) || 2889 calls.Add(call->name, call)) { 2890 delete call; 2891 return false; 2892 } 2893 } 2894 } 2895 if (!DeserializeAttributes(parser, service->attributes())) 2896 return false; 2897 DeserializeDoc(doc_comment, service->documentation()); 2898 return true; 2899 } 2900 2901 Offset<reflection::Enum> EnumDef::Serialize(FlatBufferBuilder *builder, 2902 const Parser &parser) const { 2903 std::vector<Offset<reflection::EnumVal>> enumval_offsets; 2904 for (auto it = vals.vec.begin(); it != vals.vec.end(); ++it) { 2905 enumval_offsets.push_back((*it)->Serialize(builder, parser)); 2906 } 2907 auto qualified_name = defined_namespace->GetFullyQualifiedName(name); 2908 auto name__ = builder->CreateString(qualified_name); 2909 auto vals__ = builder->CreateVector(enumval_offsets); 2910 auto type__ = underlying_type.Serialize(builder); 2911 auto attr__ = SerializeAttributes(builder, parser); 2912 auto docs__ = parser.opts.binary_schema_comments 2913 ? builder->CreateVectorOfStrings(doc_comment) 2914 : 0; 2915 return reflection::CreateEnum(*builder, name__, vals__, is_union, type__, 2916 attr__, docs__); 2917 } 2918 2919 bool EnumDef::Deserialize(Parser &parser, const reflection::Enum *_enum) { 2920 name = parser.UnqualifiedName(_enum->name()->str()); 2921 for (uoffset_t i = 0; i < _enum->values()->size(); ++i) { 2922 auto val = new EnumVal(); 2923 if (!val->Deserialize(parser, _enum->values()->Get(i)) || 2924 vals.Add(val->name, val)) { 2925 delete val; 2926 return false; 2927 } 2928 } 2929 is_union = _enum->is_union(); 2930 if (!underlying_type.Deserialize(parser, _enum->underlying_type())) { 2931 return false; 2932 } 2933 if (!DeserializeAttributes(parser, _enum->attributes())) 2934 return false; 2935 DeserializeDoc(doc_comment, _enum->documentation()); 2936 return true; 2937 } 2938 2939 Offset<reflection::EnumVal> EnumVal::Serialize(FlatBufferBuilder *builder, 2940 const Parser &parser) const { 2941 auto name__ = builder->CreateString(name); 2942 auto type__ = union_type.Serialize(builder); 2943 auto docs__ = parser.opts.binary_schema_comments 2944 ? builder->CreateVectorOfStrings(doc_comment) 2945 : 0; 2946 return reflection::CreateEnumVal(*builder, name__, value, 2947 union_type.struct_def ? union_type.struct_def->serialized_location : 0, 2948 type__, docs__); 2949 } 2950 2951 bool EnumVal::Deserialize(const Parser &parser, 2952 const reflection::EnumVal *val) { 2953 name = val->name()->str(); 2954 value = val->value(); 2955 if (!union_type.Deserialize(parser, val->union_type())) 2956 return false; 2957 DeserializeDoc(doc_comment, val->documentation()); 2958 return true; 2959 } 2960 2961 Offset<reflection::Type> Type::Serialize(FlatBufferBuilder *builder) const { 2962 return reflection::CreateType( 2963 *builder, 2964 static_cast<reflection::BaseType>(base_type), 2965 static_cast<reflection::BaseType>(element), 2966 struct_def ? struct_def->index : (enum_def ? enum_def->index : -1)); 2967 } 2968 2969 bool Type::Deserialize(const Parser &parser, const reflection::Type *type) { 2970 if (type == nullptr) return true; 2971 base_type = static_cast<BaseType>(type->base_type()); 2972 element = static_cast<BaseType>(type->element()); 2973 if (type->index() >= 0) { 2974 if (type->base_type() == reflection::Obj || 2975 (type->base_type() == reflection::Vector && 2976 type->element() == reflection::Obj)) { 2977 if (static_cast<size_t>(type->index()) < parser.structs_.vec.size()) { 2978 struct_def = parser.structs_.vec[type->index()]; 2979 struct_def->refcount++; 2980 } else { 2981 return false; 2982 } 2983 } else { 2984 if (static_cast<size_t>(type->index()) < parser.enums_.vec.size()) { 2985 enum_def = parser.enums_.vec[type->index()]; 2986 } else { 2987 return false; 2988 } 2989 } 2990 } 2991 return true; 2992 } 2993 2994 flatbuffers::Offset< 2995 flatbuffers::Vector<flatbuffers::Offset<reflection::KeyValue>>> 2996 Definition::SerializeAttributes(FlatBufferBuilder *builder, 2997 const Parser &parser) const { 2998 std::vector<flatbuffers::Offset<reflection::KeyValue>> attrs; 2999 for (auto kv = attributes.dict.begin(); kv != attributes.dict.end(); ++kv) { 3000 auto it = parser.known_attributes_.find(kv->first); 3001 FLATBUFFERS_ASSERT(it != parser.known_attributes_.end()); 3002 if (parser.opts.binary_schema_builtins || !it->second) { 3003 auto key = builder->CreateString(kv->first); 3004 auto val = builder->CreateString(kv->second->constant); 3005 attrs.push_back(reflection::CreateKeyValue(*builder, key, val)); 3006 } 3007 } 3008 if (attrs.size()) { 3009 return builder->CreateVectorOfSortedTables(&attrs); 3010 } else { 3011 return 0; 3012 } 3013 } 3014 3015 bool Definition::DeserializeAttributes( 3016 Parser &parser, const Vector<Offset<reflection::KeyValue>> *attrs) { 3017 if (attrs == nullptr) 3018 return true; 3019 for (uoffset_t i = 0; i < attrs->size(); ++i) { 3020 auto kv = attrs->Get(i); 3021 auto value = new Value(); 3022 if (kv->value()) { value->constant = kv->value()->str(); } 3023 if (attributes.Add(kv->key()->str(), value)) { 3024 delete value; 3025 return false; 3026 } 3027 parser.known_attributes_[kv->key()->str()]; 3028 } 3029 return true; 3030 } 3031 3032 /************************************************************************/ 3033 /* DESERIALIZATION */ 3034 /************************************************************************/ 3035 bool Parser::Deserialize(const uint8_t *buf, const size_t size) { 3036 flatbuffers::Verifier verifier(reinterpret_cast<const uint8_t *>(buf), size); 3037 bool size_prefixed = false; 3038 if(!reflection::SchemaBufferHasIdentifier(buf)) { 3039 if (!flatbuffers::BufferHasIdentifier(buf, reflection::SchemaIdentifier(), 3040 true)) 3041 return false; 3042 else 3043 size_prefixed = true; 3044 } 3045 auto verify_fn = size_prefixed ? &reflection::VerifySizePrefixedSchemaBuffer 3046 : &reflection::VerifySchemaBuffer; 3047 if (!verify_fn(verifier)) { 3048 return false; 3049 } 3050 auto schema = size_prefixed ? reflection::GetSizePrefixedSchema(buf) 3051 : reflection::GetSchema(buf); 3052 return Deserialize(schema); 3053 } 3054 3055 bool Parser::Deserialize(const reflection::Schema *schema) { 3056 file_identifier_ = schema->file_ident() ? schema->file_ident()->str() : ""; 3057 file_extension_ = schema->file_ext() ? schema->file_ext()->str() : ""; 3058 std::map<std::string, Namespace *> namespaces_index; 3059 3060 // Create defs without deserializing so references from fields to structs and 3061 // enums can be resolved. 3062 for (auto it = schema->objects()->begin(); it != schema->objects()->end(); 3063 ++it) { 3064 auto struct_def = new StructDef(); 3065 if (structs_.Add(it->name()->str(), struct_def)) { 3066 delete struct_def; 3067 return false; 3068 } 3069 auto type = new Type(BASE_TYPE_STRUCT, struct_def, nullptr); 3070 if (types_.Add(it->name()->str(), type)) { 3071 delete type; 3072 return false; 3073 } 3074 } 3075 for (auto it = schema->enums()->begin(); it != schema->enums()->end(); ++it) { 3076 auto enum_def = new EnumDef(); 3077 if (enums_.Add(it->name()->str(), enum_def)) { 3078 delete enum_def; 3079 return false; 3080 } 3081 auto type = new Type(BASE_TYPE_UNION, nullptr, enum_def); 3082 if (types_.Add(it->name()->str(), type)) { 3083 delete type; 3084 return false; 3085 } 3086 } 3087 3088 // Now fields can refer to structs and enums by index. 3089 for (auto it = schema->objects()->begin(); it != schema->objects()->end(); 3090 ++it) { 3091 std::string qualified_name = it->name()->str(); 3092 auto struct_def = structs_.Lookup(qualified_name); 3093 struct_def->defined_namespace = 3094 GetNamespace(qualified_name, namespaces_, namespaces_index); 3095 if (!struct_def->Deserialize(*this, * it)) { return false; } 3096 if (schema->root_table() == *it) { root_struct_def_ = struct_def; } 3097 } 3098 for (auto it = schema->enums()->begin(); it != schema->enums()->end(); ++it) { 3099 std::string qualified_name = it->name()->str(); 3100 auto enum_def = enums_.Lookup(qualified_name); 3101 enum_def->defined_namespace = 3102 GetNamespace(qualified_name, namespaces_, namespaces_index); 3103 if (!enum_def->Deserialize(*this, *it)) { return false; } 3104 } 3105 3106 if (schema->services()) { 3107 for (auto it = schema->services()->begin(); it != schema->services()->end(); 3108 ++it) { 3109 std::string qualified_name = it->name()->str(); 3110 auto service_def = new ServiceDef(); 3111 service_def->defined_namespace = 3112 GetNamespace(qualified_name, namespaces_, namespaces_index); 3113 if (!service_def->Deserialize(*this, *it) || 3114 services_.Add(qualified_name, service_def)) { 3115 delete service_def; 3116 return false; 3117 } 3118 } 3119 } 3120 3121 return true; 3122 } 3123 3124 std::string Parser::ConformTo(const Parser &base) { 3125 for (auto sit = structs_.vec.begin(); sit != structs_.vec.end(); ++sit) { 3126 auto &struct_def = **sit; 3127 auto qualified_name = 3128 struct_def.defined_namespace->GetFullyQualifiedName(struct_def.name); 3129 auto struct_def_base = base.LookupStruct(qualified_name); 3130 if (!struct_def_base) continue; 3131 for (auto fit = struct_def.fields.vec.begin(); 3132 fit != struct_def.fields.vec.end(); ++fit) { 3133 auto &field = **fit; 3134 auto field_base = struct_def_base->fields.Lookup(field.name); 3135 if (field_base) { 3136 if (field.value.offset != field_base->value.offset) 3137 return "offsets differ for field: " + field.name; 3138 if (field.value.constant != field_base->value.constant) 3139 return "defaults differ for field: " + field.name; 3140 if (!EqualByName(field.value.type, field_base->value.type)) 3141 return "types differ for field: " + field.name; 3142 } else { 3143 // Doesn't have to exist, deleting fields is fine. 3144 // But we should check if there is a field that has the same offset 3145 // but is incompatible (in the case of field renaming). 3146 for (auto fbit = struct_def_base->fields.vec.begin(); 3147 fbit != struct_def_base->fields.vec.end(); ++fbit) { 3148 field_base = *fbit; 3149 if (field.value.offset == field_base->value.offset) { 3150 if (!EqualByName(field.value.type, field_base->value.type)) 3151 return "field renamed to different type: " + field.name; 3152 break; 3153 } 3154 } 3155 } 3156 } 3157 } 3158 for (auto eit = enums_.vec.begin(); eit != enums_.vec.end(); ++eit) { 3159 auto &enum_def = **eit; 3160 auto qualified_name = 3161 enum_def.defined_namespace->GetFullyQualifiedName(enum_def.name); 3162 auto enum_def_base = base.enums_.Lookup(qualified_name); 3163 if (!enum_def_base) continue; 3164 for (auto evit = enum_def.vals.vec.begin(); evit != enum_def.vals.vec.end(); 3165 ++evit) { 3166 auto &enum_val = **evit; 3167 auto enum_val_base = enum_def_base->vals.Lookup(enum_val.name); 3168 if (enum_val_base) { 3169 if (enum_val.value != enum_val_base->value) 3170 return "values differ for enum: " + enum_val.name; 3171 } 3172 } 3173 } 3174 return ""; 3175 } 3176 3177 } // namespace flatbuffers 3178
