1 //===--- YAMLParser.cpp - Simple YAML parser ------------------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file implements a YAML parser. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Support/YAMLParser.h" 15 16 #include "llvm/ADT/ilist.h" 17 #include "llvm/ADT/ilist_node.h" 18 #include "llvm/ADT/SmallVector.h" 19 #include "llvm/ADT/StringExtras.h" 20 #include "llvm/ADT/Twine.h" 21 #include "llvm/Support/ErrorHandling.h" 22 #include "llvm/Support/MemoryBuffer.h" 23 #include "llvm/Support/raw_ostream.h" 24 #include "llvm/Support/SourceMgr.h" 25 26 using namespace llvm; 27 using namespace yaml; 28 29 enum UnicodeEncodingForm { 30 UEF_UTF32_LE, //< UTF-32 Little Endian 31 UEF_UTF32_BE, //< UTF-32 Big Endian 32 UEF_UTF16_LE, //< UTF-16 Little Endian 33 UEF_UTF16_BE, //< UTF-16 Big Endian 34 UEF_UTF8, //< UTF-8 or ascii. 35 UEF_Unknown //< Not a valid Unicode encoding. 36 }; 37 38 /// EncodingInfo - Holds the encoding type and length of the byte order mark if 39 /// it exists. Length is in {0, 2, 3, 4}. 40 typedef std::pair<UnicodeEncodingForm, unsigned> EncodingInfo; 41 42 /// getUnicodeEncoding - Reads up to the first 4 bytes to determine the Unicode 43 /// encoding form of \a Input. 44 /// 45 /// @param Input A string of length 0 or more. 46 /// @returns An EncodingInfo indicating the Unicode encoding form of the input 47 /// and how long the byte order mark is if one exists. 48 static EncodingInfo getUnicodeEncoding(StringRef Input) { 49 if (Input.size() == 0) 50 return std::make_pair(UEF_Unknown, 0); 51 52 switch (uint8_t(Input[0])) { 53 case 0x00: 54 if (Input.size() >= 4) { 55 if ( Input[1] == 0 56 && uint8_t(Input[2]) == 0xFE 57 && uint8_t(Input[3]) == 0xFF) 58 return std::make_pair(UEF_UTF32_BE, 4); 59 if (Input[1] == 0 && Input[2] == 0 && Input[3] != 0) 60 return std::make_pair(UEF_UTF32_BE, 0); 61 } 62 63 if (Input.size() >= 2 && Input[1] != 0) 64 return std::make_pair(UEF_UTF16_BE, 0); 65 return std::make_pair(UEF_Unknown, 0); 66 case 0xFF: 67 if ( Input.size() >= 4 68 && uint8_t(Input[1]) == 0xFE 69 && Input[2] == 0 70 && Input[3] == 0) 71 return std::make_pair(UEF_UTF32_LE, 4); 72 73 if (Input.size() >= 2 && uint8_t(Input[1]) == 0xFE) 74 return std::make_pair(UEF_UTF16_LE, 2); 75 return std::make_pair(UEF_Unknown, 0); 76 case 0xFE: 77 if (Input.size() >= 2 && uint8_t(Input[1]) == 0xFF) 78 return std::make_pair(UEF_UTF16_BE, 2); 79 return std::make_pair(UEF_Unknown, 0); 80 case 0xEF: 81 if ( Input.size() >= 3 82 && uint8_t(Input[1]) == 0xBB 83 && uint8_t(Input[2]) == 0xBF) 84 return std::make_pair(UEF_UTF8, 3); 85 return std::make_pair(UEF_Unknown, 0); 86 } 87 88 // It could still be utf-32 or utf-16. 89 if (Input.size() >= 4 && Input[1] == 0 && Input[2] == 0 && Input[3] == 0) 90 return std::make_pair(UEF_UTF32_LE, 0); 91 92 if (Input.size() >= 2 && Input[1] == 0) 93 return std::make_pair(UEF_UTF16_LE, 0); 94 95 return std::make_pair(UEF_UTF8, 0); 96 } 97 98 namespace llvm { 99 namespace yaml { 100 /// Token - A single YAML token. 101 struct Token : ilist_node<Token> { 102 enum TokenKind { 103 TK_Error, // Uninitialized token. 104 TK_StreamStart, 105 TK_StreamEnd, 106 TK_VersionDirective, 107 TK_TagDirective, 108 TK_DocumentStart, 109 TK_DocumentEnd, 110 TK_BlockEntry, 111 TK_BlockEnd, 112 TK_BlockSequenceStart, 113 TK_BlockMappingStart, 114 TK_FlowEntry, 115 TK_FlowSequenceStart, 116 TK_FlowSequenceEnd, 117 TK_FlowMappingStart, 118 TK_FlowMappingEnd, 119 TK_Key, 120 TK_Value, 121 TK_Scalar, 122 TK_Alias, 123 TK_Anchor, 124 TK_Tag 125 } Kind; 126 127 /// A string of length 0 or more whose begin() points to the logical location 128 /// of the token in the input. 129 StringRef Range; 130 131 Token() : Kind(TK_Error) {} 132 }; 133 } 134 } 135 136 namespace llvm { 137 template<> 138 struct ilist_sentinel_traits<Token> { 139 Token *createSentinel() const { 140 return &Sentinel; 141 } 142 static void destroySentinel(Token*) {} 143 144 Token *provideInitialHead() const { return createSentinel(); } 145 Token *ensureHead(Token*) const { return createSentinel(); } 146 static void noteHead(Token*, Token*) {} 147 148 private: 149 mutable Token Sentinel; 150 }; 151 152 template<> 153 struct ilist_node_traits<Token> { 154 Token *createNode(const Token &V) { 155 return new (Alloc.Allocate<Token>()) Token(V); 156 } 157 static void deleteNode(Token *V) {} 158 159 void addNodeToList(Token *) {} 160 void removeNodeFromList(Token *) {} 161 void transferNodesFromList(ilist_node_traits & /*SrcTraits*/, 162 ilist_iterator<Token> /*first*/, 163 ilist_iterator<Token> /*last*/) {} 164 165 BumpPtrAllocator Alloc; 166 }; 167 } 168 169 typedef ilist<Token> TokenQueueT; 170 171 namespace { 172 /// @brief This struct is used to track simple keys. 173 /// 174 /// Simple keys are handled by creating an entry in SimpleKeys for each Token 175 /// which could legally be the start of a simple key. When peekNext is called, 176 /// if the Token To be returned is referenced by a SimpleKey, we continue 177 /// tokenizing until that potential simple key has either been found to not be 178 /// a simple key (we moved on to the next line or went further than 1024 chars). 179 /// Or when we run into a Value, and then insert a Key token (and possibly 180 /// others) before the SimpleKey's Tok. 181 struct SimpleKey { 182 TokenQueueT::iterator Tok; 183 unsigned Column; 184 unsigned Line; 185 unsigned FlowLevel; 186 bool IsRequired; 187 188 bool operator ==(const SimpleKey &Other) { 189 return Tok == Other.Tok; 190 } 191 }; 192 } 193 194 /// @brief The Unicode scalar value of a UTF-8 minimal well-formed code unit 195 /// subsequence and the subsequence's length in code units (uint8_t). 196 /// A length of 0 represents an error. 197 typedef std::pair<uint32_t, unsigned> UTF8Decoded; 198 199 static UTF8Decoded decodeUTF8(StringRef Range) { 200 StringRef::iterator Position= Range.begin(); 201 StringRef::iterator End = Range.end(); 202 // 1 byte: [0x00, 0x7f] 203 // Bit pattern: 0xxxxxxx 204 if ((*Position & 0x80) == 0) { 205 return std::make_pair(*Position, 1); 206 } 207 // 2 bytes: [0x80, 0x7ff] 208 // Bit pattern: 110xxxxx 10xxxxxx 209 if (Position + 1 != End && 210 ((*Position & 0xE0) == 0xC0) && 211 ((*(Position + 1) & 0xC0) == 0x80)) { 212 uint32_t codepoint = ((*Position & 0x1F) << 6) | 213 (*(Position + 1) & 0x3F); 214 if (codepoint >= 0x80) 215 return std::make_pair(codepoint, 2); 216 } 217 // 3 bytes: [0x8000, 0xffff] 218 // Bit pattern: 1110xxxx 10xxxxxx 10xxxxxx 219 if (Position + 2 != End && 220 ((*Position & 0xF0) == 0xE0) && 221 ((*(Position + 1) & 0xC0) == 0x80) && 222 ((*(Position + 2) & 0xC0) == 0x80)) { 223 uint32_t codepoint = ((*Position & 0x0F) << 12) | 224 ((*(Position + 1) & 0x3F) << 6) | 225 (*(Position + 2) & 0x3F); 226 // Codepoints between 0xD800 and 0xDFFF are invalid, as 227 // they are high / low surrogate halves used by UTF-16. 228 if (codepoint >= 0x800 && 229 (codepoint < 0xD800 || codepoint > 0xDFFF)) 230 return std::make_pair(codepoint, 3); 231 } 232 // 4 bytes: [0x10000, 0x10FFFF] 233 // Bit pattern: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx 234 if (Position + 3 != End && 235 ((*Position & 0xF8) == 0xF0) && 236 ((*(Position + 1) & 0xC0) == 0x80) && 237 ((*(Position + 2) & 0xC0) == 0x80) && 238 ((*(Position + 3) & 0xC0) == 0x80)) { 239 uint32_t codepoint = ((*Position & 0x07) << 18) | 240 ((*(Position + 1) & 0x3F) << 12) | 241 ((*(Position + 2) & 0x3F) << 6) | 242 (*(Position + 3) & 0x3F); 243 if (codepoint >= 0x10000 && codepoint <= 0x10FFFF) 244 return std::make_pair(codepoint, 4); 245 } 246 return std::make_pair(0, 0); 247 } 248 249 namespace llvm { 250 namespace yaml { 251 /// @brief Scans YAML tokens from a MemoryBuffer. 252 class Scanner { 253 public: 254 Scanner(const StringRef Input, SourceMgr &SM); 255 256 /// @brief Parse the next token and return it without popping it. 257 Token &peekNext(); 258 259 /// @brief Parse the next token and pop it from the queue. 260 Token getNext(); 261 262 void printError(SMLoc Loc, SourceMgr::DiagKind Kind, const Twine &Message, 263 ArrayRef<SMRange> Ranges = ArrayRef<SMRange>()) { 264 SM.PrintMessage(Loc, Kind, Message, Ranges); 265 } 266 267 void setError(const Twine &Message, StringRef::iterator Position) { 268 if (Current >= End) 269 Current = End - 1; 270 271 // Don't print out more errors after the first one we encounter. The rest 272 // are just the result of the first, and have no meaning. 273 if (!Failed) 274 printError(SMLoc::getFromPointer(Current), SourceMgr::DK_Error, Message); 275 Failed = true; 276 } 277 278 void setError(const Twine &Message) { 279 setError(Message, Current); 280 } 281 282 /// @brief Returns true if an error occurred while parsing. 283 bool failed() { 284 return Failed; 285 } 286 287 private: 288 StringRef currentInput() { 289 return StringRef(Current, End - Current); 290 } 291 292 /// @brief Decode a UTF-8 minimal well-formed code unit subsequence starting 293 /// at \a Position. 294 /// 295 /// If the UTF-8 code units starting at Position do not form a well-formed 296 /// code unit subsequence, then the Unicode scalar value is 0, and the length 297 /// is 0. 298 UTF8Decoded decodeUTF8(StringRef::iterator Position) { 299 return ::decodeUTF8(StringRef(Position, End - Position)); 300 } 301 302 // The following functions are based on the gramar rules in the YAML spec. The 303 // style of the function names it meant to closely match how they are written 304 // in the spec. The number within the [] is the number of the grammar rule in 305 // the spec. 306 // 307 // See 4.2 [Production Naming Conventions] for the meaning of the prefixes. 308 // 309 // c- 310 // A production starting and ending with a special character. 311 // b- 312 // A production matching a single line break. 313 // nb- 314 // A production starting and ending with a non-break character. 315 // s- 316 // A production starting and ending with a white space character. 317 // ns- 318 // A production starting and ending with a non-space character. 319 // l- 320 // A production matching complete line(s). 321 322 /// @brief Skip a single nb-char[27] starting at Position. 323 /// 324 /// A nb-char is 0x9 | [0x20-0x7E] | 0x85 | [0xA0-0xD7FF] | [0xE000-0xFEFE] 325 /// | [0xFF00-0xFFFD] | [0x10000-0x10FFFF] 326 /// 327 /// @returns The code unit after the nb-char, or Position if it's not an 328 /// nb-char. 329 StringRef::iterator skip_nb_char(StringRef::iterator Position); 330 331 /// @brief Skip a single b-break[28] starting at Position. 332 /// 333 /// A b-break is 0xD 0xA | 0xD | 0xA 334 /// 335 /// @returns The code unit after the b-break, or Position if it's not a 336 /// b-break. 337 StringRef::iterator skip_b_break(StringRef::iterator Position); 338 339 /// @brief Skip a single s-white[33] starting at Position. 340 /// 341 /// A s-white is 0x20 | 0x9 342 /// 343 /// @returns The code unit after the s-white, or Position if it's not a 344 /// s-white. 345 StringRef::iterator skip_s_white(StringRef::iterator Position); 346 347 /// @brief Skip a single ns-char[34] starting at Position. 348 /// 349 /// A ns-char is nb-char - s-white 350 /// 351 /// @returns The code unit after the ns-char, or Position if it's not a 352 /// ns-char. 353 StringRef::iterator skip_ns_char(StringRef::iterator Position); 354 355 typedef StringRef::iterator (Scanner::*SkipWhileFunc)(StringRef::iterator); 356 /// @brief Skip minimal well-formed code unit subsequences until Func 357 /// returns its input. 358 /// 359 /// @returns The code unit after the last minimal well-formed code unit 360 /// subsequence that Func accepted. 361 StringRef::iterator skip_while( SkipWhileFunc Func 362 , StringRef::iterator Position); 363 364 /// @brief Scan ns-uri-char[39]s starting at Cur. 365 /// 366 /// This updates Cur and Column while scanning. 367 /// 368 /// @returns A StringRef starting at Cur which covers the longest contiguous 369 /// sequence of ns-uri-char. 370 StringRef scan_ns_uri_char(); 371 372 /// @brief Scan ns-plain-one-line[133] starting at \a Cur. 373 StringRef scan_ns_plain_one_line(); 374 375 /// @brief Consume a minimal well-formed code unit subsequence starting at 376 /// \a Cur. Return false if it is not the same Unicode scalar value as 377 /// \a Expected. This updates \a Column. 378 bool consume(uint32_t Expected); 379 380 /// @brief Skip \a Distance UTF-8 code units. Updates \a Cur and \a Column. 381 void skip(uint32_t Distance); 382 383 /// @brief Return true if the minimal well-formed code unit subsequence at 384 /// Pos is whitespace or a new line 385 bool isBlankOrBreak(StringRef::iterator Position); 386 387 /// @brief If IsSimpleKeyAllowed, create and push_back a new SimpleKey. 388 void saveSimpleKeyCandidate( TokenQueueT::iterator Tok 389 , unsigned AtColumn 390 , bool IsRequired); 391 392 /// @brief Remove simple keys that can no longer be valid simple keys. 393 /// 394 /// Invalid simple keys are not on the current line or are further than 1024 395 /// columns back. 396 void removeStaleSimpleKeyCandidates(); 397 398 /// @brief Remove all simple keys on FlowLevel \a Level. 399 void removeSimpleKeyCandidatesOnFlowLevel(unsigned Level); 400 401 /// @brief Unroll indentation in \a Indents back to \a Col. Creates BlockEnd 402 /// tokens if needed. 403 bool unrollIndent(int ToColumn); 404 405 /// @brief Increase indent to \a Col. Creates \a Kind token at \a InsertPoint 406 /// if needed. 407 bool rollIndent( int ToColumn 408 , Token::TokenKind Kind 409 , TokenQueueT::iterator InsertPoint); 410 411 /// @brief Skip whitespace and comments until the start of the next token. 412 void scanToNextToken(); 413 414 /// @brief Must be the first token generated. 415 bool scanStreamStart(); 416 417 /// @brief Generate tokens needed to close out the stream. 418 bool scanStreamEnd(); 419 420 /// @brief Scan a %BLAH directive. 421 bool scanDirective(); 422 423 /// @brief Scan a ... or ---. 424 bool scanDocumentIndicator(bool IsStart); 425 426 /// @brief Scan a [ or { and generate the proper flow collection start token. 427 bool scanFlowCollectionStart(bool IsSequence); 428 429 /// @brief Scan a ] or } and generate the proper flow collection end token. 430 bool scanFlowCollectionEnd(bool IsSequence); 431 432 /// @brief Scan the , that separates entries in a flow collection. 433 bool scanFlowEntry(); 434 435 /// @brief Scan the - that starts block sequence entries. 436 bool scanBlockEntry(); 437 438 /// @brief Scan an explicit ? indicating a key. 439 bool scanKey(); 440 441 /// @brief Scan an explicit : indicating a value. 442 bool scanValue(); 443 444 /// @brief Scan a quoted scalar. 445 bool scanFlowScalar(bool IsDoubleQuoted); 446 447 /// @brief Scan an unquoted scalar. 448 bool scanPlainScalar(); 449 450 /// @brief Scan an Alias or Anchor starting with * or &. 451 bool scanAliasOrAnchor(bool IsAlias); 452 453 /// @brief Scan a block scalar starting with | or >. 454 bool scanBlockScalar(bool IsLiteral); 455 456 /// @brief Scan a tag of the form !stuff. 457 bool scanTag(); 458 459 /// @brief Dispatch to the next scanning function based on \a *Cur. 460 bool fetchMoreTokens(); 461 462 /// @brief The SourceMgr used for diagnostics and buffer management. 463 SourceMgr &SM; 464 465 /// @brief The original input. 466 MemoryBuffer *InputBuffer; 467 468 /// @brief The current position of the scanner. 469 StringRef::iterator Current; 470 471 /// @brief The end of the input (one past the last character). 472 StringRef::iterator End; 473 474 /// @brief Current YAML indentation level in spaces. 475 int Indent; 476 477 /// @brief Current column number in Unicode code points. 478 unsigned Column; 479 480 /// @brief Current line number. 481 unsigned Line; 482 483 /// @brief How deep we are in flow style containers. 0 Means at block level. 484 unsigned FlowLevel; 485 486 /// @brief Are we at the start of the stream? 487 bool IsStartOfStream; 488 489 /// @brief Can the next token be the start of a simple key? 490 bool IsSimpleKeyAllowed; 491 492 /// @brief Is the next token required to start a simple key? 493 bool IsSimpleKeyRequired; 494 495 /// @brief True if an error has occurred. 496 bool Failed; 497 498 /// @brief Queue of tokens. This is required to queue up tokens while looking 499 /// for the end of a simple key. And for cases where a single character 500 /// can produce multiple tokens (e.g. BlockEnd). 501 TokenQueueT TokenQueue; 502 503 /// @brief Indentation levels. 504 SmallVector<int, 4> Indents; 505 506 /// @brief Potential simple keys. 507 SmallVector<SimpleKey, 4> SimpleKeys; 508 }; 509 510 } // end namespace yaml 511 } // end namespace llvm 512 513 /// encodeUTF8 - Encode \a UnicodeScalarValue in UTF-8 and append it to result. 514 static void encodeUTF8( uint32_t UnicodeScalarValue 515 , SmallVectorImpl<char> &Result) { 516 if (UnicodeScalarValue <= 0x7F) { 517 Result.push_back(UnicodeScalarValue & 0x7F); 518 } else if (UnicodeScalarValue <= 0x7FF) { 519 uint8_t FirstByte = 0xC0 | ((UnicodeScalarValue & 0x7C0) >> 6); 520 uint8_t SecondByte = 0x80 | (UnicodeScalarValue & 0x3F); 521 Result.push_back(FirstByte); 522 Result.push_back(SecondByte); 523 } else if (UnicodeScalarValue <= 0xFFFF) { 524 uint8_t FirstByte = 0xE0 | ((UnicodeScalarValue & 0xF000) >> 12); 525 uint8_t SecondByte = 0x80 | ((UnicodeScalarValue & 0xFC0) >> 6); 526 uint8_t ThirdByte = 0x80 | (UnicodeScalarValue & 0x3F); 527 Result.push_back(FirstByte); 528 Result.push_back(SecondByte); 529 Result.push_back(ThirdByte); 530 } else if (UnicodeScalarValue <= 0x10FFFF) { 531 uint8_t FirstByte = 0xF0 | ((UnicodeScalarValue & 0x1F0000) >> 18); 532 uint8_t SecondByte = 0x80 | ((UnicodeScalarValue & 0x3F000) >> 12); 533 uint8_t ThirdByte = 0x80 | ((UnicodeScalarValue & 0xFC0) >> 6); 534 uint8_t FourthByte = 0x80 | (UnicodeScalarValue & 0x3F); 535 Result.push_back(FirstByte); 536 Result.push_back(SecondByte); 537 Result.push_back(ThirdByte); 538 Result.push_back(FourthByte); 539 } 540 } 541 542 bool yaml::dumpTokens(StringRef Input, raw_ostream &OS) { 543 SourceMgr SM; 544 Scanner scanner(Input, SM); 545 while (true) { 546 Token T = scanner.getNext(); 547 switch (T.Kind) { 548 case Token::TK_StreamStart: 549 OS << "Stream-Start: "; 550 break; 551 case Token::TK_StreamEnd: 552 OS << "Stream-End: "; 553 break; 554 case Token::TK_VersionDirective: 555 OS << "Version-Directive: "; 556 break; 557 case Token::TK_TagDirective: 558 OS << "Tag-Directive: "; 559 break; 560 case Token::TK_DocumentStart: 561 OS << "Document-Start: "; 562 break; 563 case Token::TK_DocumentEnd: 564 OS << "Document-End: "; 565 break; 566 case Token::TK_BlockEntry: 567 OS << "Block-Entry: "; 568 break; 569 case Token::TK_BlockEnd: 570 OS << "Block-End: "; 571 break; 572 case Token::TK_BlockSequenceStart: 573 OS << "Block-Sequence-Start: "; 574 break; 575 case Token::TK_BlockMappingStart: 576 OS << "Block-Mapping-Start: "; 577 break; 578 case Token::TK_FlowEntry: 579 OS << "Flow-Entry: "; 580 break; 581 case Token::TK_FlowSequenceStart: 582 OS << "Flow-Sequence-Start: "; 583 break; 584 case Token::TK_FlowSequenceEnd: 585 OS << "Flow-Sequence-End: "; 586 break; 587 case Token::TK_FlowMappingStart: 588 OS << "Flow-Mapping-Start: "; 589 break; 590 case Token::TK_FlowMappingEnd: 591 OS << "Flow-Mapping-End: "; 592 break; 593 case Token::TK_Key: 594 OS << "Key: "; 595 break; 596 case Token::TK_Value: 597 OS << "Value: "; 598 break; 599 case Token::TK_Scalar: 600 OS << "Scalar: "; 601 break; 602 case Token::TK_Alias: 603 OS << "Alias: "; 604 break; 605 case Token::TK_Anchor: 606 OS << "Anchor: "; 607 break; 608 case Token::TK_Tag: 609 OS << "Tag: "; 610 break; 611 case Token::TK_Error: 612 break; 613 } 614 OS << T.Range << "\n"; 615 if (T.Kind == Token::TK_StreamEnd) 616 break; 617 else if (T.Kind == Token::TK_Error) 618 return false; 619 } 620 return true; 621 } 622 623 bool yaml::scanTokens(StringRef Input) { 624 llvm::SourceMgr SM; 625 llvm::yaml::Scanner scanner(Input, SM); 626 for (;;) { 627 llvm::yaml::Token T = scanner.getNext(); 628 if (T.Kind == Token::TK_StreamEnd) 629 break; 630 else if (T.Kind == Token::TK_Error) 631 return false; 632 } 633 return true; 634 } 635 636 std::string yaml::escape(StringRef Input) { 637 std::string EscapedInput; 638 for (StringRef::iterator i = Input.begin(), e = Input.end(); i != e; ++i) { 639 if (*i == '\\') 640 EscapedInput += "\\\\"; 641 else if (*i == '"') 642 EscapedInput += "\\\""; 643 else if (*i == 0) 644 EscapedInput += "\\0"; 645 else if (*i == 0x07) 646 EscapedInput += "\\a"; 647 else if (*i == 0x08) 648 EscapedInput += "\\b"; 649 else if (*i == 0x09) 650 EscapedInput += "\\t"; 651 else if (*i == 0x0A) 652 EscapedInput += "\\n"; 653 else if (*i == 0x0B) 654 EscapedInput += "\\v"; 655 else if (*i == 0x0C) 656 EscapedInput += "\\f"; 657 else if (*i == 0x0D) 658 EscapedInput += "\\r"; 659 else if (*i == 0x1B) 660 EscapedInput += "\\e"; 661 else if (*i >= 0 && *i < 0x20) { // Control characters not handled above. 662 std::string HexStr = utohexstr(*i); 663 EscapedInput += "\\x" + std::string(2 - HexStr.size(), '0') + HexStr; 664 } else if (*i & 0x80) { // UTF-8 multiple code unit subsequence. 665 UTF8Decoded UnicodeScalarValue 666 = decodeUTF8(StringRef(i, Input.end() - i)); 667 if (UnicodeScalarValue.second == 0) { 668 // Found invalid char. 669 SmallString<4> Val; 670 encodeUTF8(0xFFFD, Val); 671 EscapedInput.insert(EscapedInput.end(), Val.begin(), Val.end()); 672 // FIXME: Error reporting. 673 return EscapedInput; 674 } 675 if (UnicodeScalarValue.first == 0x85) 676 EscapedInput += "\\N"; 677 else if (UnicodeScalarValue.first == 0xA0) 678 EscapedInput += "\\_"; 679 else if (UnicodeScalarValue.first == 0x2028) 680 EscapedInput += "\\L"; 681 else if (UnicodeScalarValue.first == 0x2029) 682 EscapedInput += "\\P"; 683 else { 684 std::string HexStr = utohexstr(UnicodeScalarValue.first); 685 if (HexStr.size() <= 2) 686 EscapedInput += "\\x" + std::string(2 - HexStr.size(), '0') + HexStr; 687 else if (HexStr.size() <= 4) 688 EscapedInput += "\\u" + std::string(4 - HexStr.size(), '0') + HexStr; 689 else if (HexStr.size() <= 8) 690 EscapedInput += "\\U" + std::string(8 - HexStr.size(), '0') + HexStr; 691 } 692 i += UnicodeScalarValue.second - 1; 693 } else 694 EscapedInput.push_back(*i); 695 } 696 return EscapedInput; 697 } 698 699 Scanner::Scanner(StringRef Input, SourceMgr &sm) 700 : SM(sm) 701 , Indent(-1) 702 , Column(0) 703 , Line(0) 704 , FlowLevel(0) 705 , IsStartOfStream(true) 706 , IsSimpleKeyAllowed(true) 707 , IsSimpleKeyRequired(false) 708 , Failed(false) { 709 InputBuffer = MemoryBuffer::getMemBuffer(Input, "YAML"); 710 SM.AddNewSourceBuffer(InputBuffer, SMLoc()); 711 Current = InputBuffer->getBufferStart(); 712 End = InputBuffer->getBufferEnd(); 713 } 714 715 Token &Scanner::peekNext() { 716 // If the current token is a possible simple key, keep parsing until we 717 // can confirm. 718 bool NeedMore = false; 719 while (true) { 720 if (TokenQueue.empty() || NeedMore) { 721 if (!fetchMoreTokens()) { 722 TokenQueue.clear(); 723 TokenQueue.push_back(Token()); 724 return TokenQueue.front(); 725 } 726 } 727 assert(!TokenQueue.empty() && 728 "fetchMoreTokens lied about getting tokens!"); 729 730 removeStaleSimpleKeyCandidates(); 731 SimpleKey SK; 732 SK.Tok = TokenQueue.front(); 733 if (std::find(SimpleKeys.begin(), SimpleKeys.end(), SK) 734 == SimpleKeys.end()) 735 break; 736 else 737 NeedMore = true; 738 } 739 return TokenQueue.front(); 740 } 741 742 Token Scanner::getNext() { 743 Token Ret = peekNext(); 744 // TokenQueue can be empty if there was an error getting the next token. 745 if (!TokenQueue.empty()) 746 TokenQueue.pop_front(); 747 748 // There cannot be any referenced Token's if the TokenQueue is empty. So do a 749 // quick deallocation of them all. 750 if (TokenQueue.empty()) { 751 TokenQueue.Alloc.Reset(); 752 } 753 754 return Ret; 755 } 756 757 StringRef::iterator Scanner::skip_nb_char(StringRef::iterator Position) { 758 // Check 7 bit c-printable - b-char. 759 if ( *Position == 0x09 760 || (*Position >= 0x20 && *Position <= 0x7E)) 761 return Position + 1; 762 763 // Check for valid UTF-8. 764 if (uint8_t(*Position) & 0x80) { 765 UTF8Decoded u8d = decodeUTF8(Position); 766 if ( u8d.second != 0 767 && u8d.first != 0xFEFF 768 && ( u8d.first == 0x85 769 || ( u8d.first >= 0xA0 770 && u8d.first <= 0xD7FF) 771 || ( u8d.first >= 0xE000 772 && u8d.first <= 0xFFFD) 773 || ( u8d.first >= 0x10000 774 && u8d.first <= 0x10FFFF))) 775 return Position + u8d.second; 776 } 777 return Position; 778 } 779 780 StringRef::iterator Scanner::skip_b_break(StringRef::iterator Position) { 781 if (*Position == 0x0D) { 782 if (Position + 1 != End && *(Position + 1) == 0x0A) 783 return Position + 2; 784 return Position + 1; 785 } 786 787 if (*Position == 0x0A) 788 return Position + 1; 789 return Position; 790 } 791 792 793 StringRef::iterator Scanner::skip_s_white(StringRef::iterator Position) { 794 if (Position == End) 795 return Position; 796 if (*Position == ' ' || *Position == '\t') 797 return Position + 1; 798 return Position; 799 } 800 801 StringRef::iterator Scanner::skip_ns_char(StringRef::iterator Position) { 802 if (Position == End) 803 return Position; 804 if (*Position == ' ' || *Position == '\t') 805 return Position; 806 return skip_nb_char(Position); 807 } 808 809 StringRef::iterator Scanner::skip_while( SkipWhileFunc Func 810 , StringRef::iterator Position) { 811 while (true) { 812 StringRef::iterator i = (this->*Func)(Position); 813 if (i == Position) 814 break; 815 Position = i; 816 } 817 return Position; 818 } 819 820 static bool is_ns_hex_digit(const char C) { 821 return (C >= '0' && C <= '9') 822 || (C >= 'a' && C <= 'z') 823 || (C >= 'A' && C <= 'Z'); 824 } 825 826 static bool is_ns_word_char(const char C) { 827 return C == '-' 828 || (C >= 'a' && C <= 'z') 829 || (C >= 'A' && C <= 'Z'); 830 } 831 832 StringRef Scanner::scan_ns_uri_char() { 833 StringRef::iterator Start = Current; 834 while (true) { 835 if (Current == End) 836 break; 837 if (( *Current == '%' 838 && Current + 2 < End 839 && is_ns_hex_digit(*(Current + 1)) 840 && is_ns_hex_digit(*(Current + 2))) 841 || is_ns_word_char(*Current) 842 || StringRef(Current, 1).find_first_of("#;/?:@&=+$,_.!~*'()[]") 843 != StringRef::npos) { 844 ++Current; 845 ++Column; 846 } else 847 break; 848 } 849 return StringRef(Start, Current - Start); 850 } 851 852 StringRef Scanner::scan_ns_plain_one_line() { 853 StringRef::iterator start = Current; 854 // The first character must already be verified. 855 ++Current; 856 while (true) { 857 if (Current == End) { 858 break; 859 } else if (*Current == ':') { 860 // Check if the next character is a ns-char. 861 if (Current + 1 == End) 862 break; 863 StringRef::iterator i = skip_ns_char(Current + 1); 864 if (Current + 1 != i) { 865 Current = i; 866 Column += 2; // Consume both the ':' and ns-char. 867 } else 868 break; 869 } else if (*Current == '#') { 870 // Check if the previous character was a ns-char. 871 // The & 0x80 check is to check for the trailing byte of a utf-8 872 if (*(Current - 1) & 0x80 || skip_ns_char(Current - 1) == Current) { 873 ++Current; 874 ++Column; 875 } else 876 break; 877 } else { 878 StringRef::iterator i = skip_nb_char(Current); 879 if (i == Current) 880 break; 881 Current = i; 882 ++Column; 883 } 884 } 885 return StringRef(start, Current - start); 886 } 887 888 bool Scanner::consume(uint32_t Expected) { 889 if (Expected >= 0x80) 890 report_fatal_error("Not dealing with this yet"); 891 if (Current == End) 892 return false; 893 if (uint8_t(*Current) >= 0x80) 894 report_fatal_error("Not dealing with this yet"); 895 if (uint8_t(*Current) == Expected) { 896 ++Current; 897 ++Column; 898 return true; 899 } 900 return false; 901 } 902 903 void Scanner::skip(uint32_t Distance) { 904 Current += Distance; 905 Column += Distance; 906 } 907 908 bool Scanner::isBlankOrBreak(StringRef::iterator Position) { 909 if (Position == End) 910 return false; 911 if ( *Position == ' ' || *Position == '\t' 912 || *Position == '\r' || *Position == '\n') 913 return true; 914 return false; 915 } 916 917 void Scanner::saveSimpleKeyCandidate( TokenQueueT::iterator Tok 918 , unsigned AtColumn 919 , bool IsRequired) { 920 if (IsSimpleKeyAllowed) { 921 SimpleKey SK; 922 SK.Tok = Tok; 923 SK.Line = Line; 924 SK.Column = AtColumn; 925 SK.IsRequired = IsRequired; 926 SK.FlowLevel = FlowLevel; 927 SimpleKeys.push_back(SK); 928 } 929 } 930 931 void Scanner::removeStaleSimpleKeyCandidates() { 932 for (SmallVectorImpl<SimpleKey>::iterator i = SimpleKeys.begin(); 933 i != SimpleKeys.end();) { 934 if (i->Line != Line || i->Column + 1024 < Column) { 935 if (i->IsRequired) 936 setError( "Could not find expected : for simple key" 937 , i->Tok->Range.begin()); 938 i = SimpleKeys.erase(i); 939 } else 940 ++i; 941 } 942 } 943 944 void Scanner::removeSimpleKeyCandidatesOnFlowLevel(unsigned Level) { 945 if (!SimpleKeys.empty() && (SimpleKeys.end() - 1)->FlowLevel == Level) 946 SimpleKeys.pop_back(); 947 } 948 949 bool Scanner::unrollIndent(int ToColumn) { 950 Token T; 951 // Indentation is ignored in flow. 952 if (FlowLevel != 0) 953 return true; 954 955 while (Indent > ToColumn) { 956 T.Kind = Token::TK_BlockEnd; 957 T.Range = StringRef(Current, 1); 958 TokenQueue.push_back(T); 959 Indent = Indents.pop_back_val(); 960 } 961 962 return true; 963 } 964 965 bool Scanner::rollIndent( int ToColumn 966 , Token::TokenKind Kind 967 , TokenQueueT::iterator InsertPoint) { 968 if (FlowLevel) 969 return true; 970 if (Indent < ToColumn) { 971 Indents.push_back(Indent); 972 Indent = ToColumn; 973 974 Token T; 975 T.Kind = Kind; 976 T.Range = StringRef(Current, 0); 977 TokenQueue.insert(InsertPoint, T); 978 } 979 return true; 980 } 981 982 void Scanner::scanToNextToken() { 983 while (true) { 984 while (*Current == ' ' || *Current == '\t') { 985 skip(1); 986 } 987 988 // Skip comment. 989 if (*Current == '#') { 990 while (true) { 991 // This may skip more than one byte, thus Column is only incremented 992 // for code points. 993 StringRef::iterator i = skip_nb_char(Current); 994 if (i == Current) 995 break; 996 Current = i; 997 ++Column; 998 } 999 } 1000 1001 // Skip EOL. 1002 StringRef::iterator i = skip_b_break(Current); 1003 if (i == Current) 1004 break; 1005 Current = i; 1006 ++Line; 1007 Column = 0; 1008 // New lines may start a simple key. 1009 if (!FlowLevel) 1010 IsSimpleKeyAllowed = true; 1011 } 1012 } 1013 1014 bool Scanner::scanStreamStart() { 1015 IsStartOfStream = false; 1016 1017 EncodingInfo EI = getUnicodeEncoding(currentInput()); 1018 1019 Token T; 1020 T.Kind = Token::TK_StreamStart; 1021 T.Range = StringRef(Current, EI.second); 1022 TokenQueue.push_back(T); 1023 Current += EI.second; 1024 return true; 1025 } 1026 1027 bool Scanner::scanStreamEnd() { 1028 // Force an ending new line if one isn't present. 1029 if (Column != 0) { 1030 Column = 0; 1031 ++Line; 1032 } 1033 1034 unrollIndent(-1); 1035 SimpleKeys.clear(); 1036 IsSimpleKeyAllowed = false; 1037 1038 Token T; 1039 T.Kind = Token::TK_StreamEnd; 1040 T.Range = StringRef(Current, 0); 1041 TokenQueue.push_back(T); 1042 return true; 1043 } 1044 1045 bool Scanner::scanDirective() { 1046 // Reset the indentation level. 1047 unrollIndent(-1); 1048 SimpleKeys.clear(); 1049 IsSimpleKeyAllowed = false; 1050 1051 StringRef::iterator Start = Current; 1052 consume('%'); 1053 StringRef::iterator NameStart = Current; 1054 Current = skip_while(&Scanner::skip_ns_char, Current); 1055 StringRef Name(NameStart, Current - NameStart); 1056 Current = skip_while(&Scanner::skip_s_white, Current); 1057 1058 if (Name == "YAML") { 1059 Current = skip_while(&Scanner::skip_ns_char, Current); 1060 Token T; 1061 T.Kind = Token::TK_VersionDirective; 1062 T.Range = StringRef(Start, Current - Start); 1063 TokenQueue.push_back(T); 1064 return true; 1065 } 1066 return false; 1067 } 1068 1069 bool Scanner::scanDocumentIndicator(bool IsStart) { 1070 unrollIndent(-1); 1071 SimpleKeys.clear(); 1072 IsSimpleKeyAllowed = false; 1073 1074 Token T; 1075 T.Kind = IsStart ? Token::TK_DocumentStart : Token::TK_DocumentEnd; 1076 T.Range = StringRef(Current, 3); 1077 skip(3); 1078 TokenQueue.push_back(T); 1079 return true; 1080 } 1081 1082 bool Scanner::scanFlowCollectionStart(bool IsSequence) { 1083 Token T; 1084 T.Kind = IsSequence ? Token::TK_FlowSequenceStart 1085 : Token::TK_FlowMappingStart; 1086 T.Range = StringRef(Current, 1); 1087 skip(1); 1088 TokenQueue.push_back(T); 1089 1090 // [ and { may begin a simple key. 1091 saveSimpleKeyCandidate(TokenQueue.back(), Column - 1, false); 1092 1093 // And may also be followed by a simple key. 1094 IsSimpleKeyAllowed = true; 1095 ++FlowLevel; 1096 return true; 1097 } 1098 1099 bool Scanner::scanFlowCollectionEnd(bool IsSequence) { 1100 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel); 1101 IsSimpleKeyAllowed = false; 1102 Token T; 1103 T.Kind = IsSequence ? Token::TK_FlowSequenceEnd 1104 : Token::TK_FlowMappingEnd; 1105 T.Range = StringRef(Current, 1); 1106 skip(1); 1107 TokenQueue.push_back(T); 1108 if (FlowLevel) 1109 --FlowLevel; 1110 return true; 1111 } 1112 1113 bool Scanner::scanFlowEntry() { 1114 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel); 1115 IsSimpleKeyAllowed = true; 1116 Token T; 1117 T.Kind = Token::TK_FlowEntry; 1118 T.Range = StringRef(Current, 1); 1119 skip(1); 1120 TokenQueue.push_back(T); 1121 return true; 1122 } 1123 1124 bool Scanner::scanBlockEntry() { 1125 rollIndent(Column, Token::TK_BlockSequenceStart, TokenQueue.end()); 1126 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel); 1127 IsSimpleKeyAllowed = true; 1128 Token T; 1129 T.Kind = Token::TK_BlockEntry; 1130 T.Range = StringRef(Current, 1); 1131 skip(1); 1132 TokenQueue.push_back(T); 1133 return true; 1134 } 1135 1136 bool Scanner::scanKey() { 1137 if (!FlowLevel) 1138 rollIndent(Column, Token::TK_BlockMappingStart, TokenQueue.end()); 1139 1140 removeSimpleKeyCandidatesOnFlowLevel(FlowLevel); 1141 IsSimpleKeyAllowed = !FlowLevel; 1142 1143 Token T; 1144 T.Kind = Token::TK_Key; 1145 T.Range = StringRef(Current, 1); 1146 skip(1); 1147 TokenQueue.push_back(T); 1148 return true; 1149 } 1150 1151 bool Scanner::scanValue() { 1152 // If the previous token could have been a simple key, insert the key token 1153 // into the token queue. 1154 if (!SimpleKeys.empty()) { 1155 SimpleKey SK = SimpleKeys.pop_back_val(); 1156 Token T; 1157 T.Kind = Token::TK_Key; 1158 T.Range = SK.Tok->Range; 1159 TokenQueueT::iterator i, e; 1160 for (i = TokenQueue.begin(), e = TokenQueue.end(); i != e; ++i) { 1161 if (i == SK.Tok) 1162 break; 1163 } 1164 assert(i != e && "SimpleKey not in token queue!"); 1165 i = TokenQueue.insert(i, T); 1166 1167 // We may also need to add a Block-Mapping-Start token. 1168 rollIndent(SK.Column, Token::TK_BlockMappingStart, i); 1169 1170 IsSimpleKeyAllowed = false; 1171 } else { 1172 if (!FlowLevel) 1173 rollIndent(Column, Token::TK_BlockMappingStart, TokenQueue.end()); 1174 IsSimpleKeyAllowed = !FlowLevel; 1175 } 1176 1177 Token T; 1178 T.Kind = Token::TK_Value; 1179 T.Range = StringRef(Current, 1); 1180 skip(1); 1181 TokenQueue.push_back(T); 1182 return true; 1183 } 1184 1185 // Forbidding inlining improves performance by roughly 20%. 1186 // FIXME: Remove once llvm optimizes this to the faster version without hints. 1187 LLVM_ATTRIBUTE_NOINLINE static bool 1188 wasEscaped(StringRef::iterator First, StringRef::iterator Position); 1189 1190 // Returns whether a character at 'Position' was escaped with a leading '\'. 1191 // 'First' specifies the position of the first character in the string. 1192 static bool wasEscaped(StringRef::iterator First, 1193 StringRef::iterator Position) { 1194 assert(Position - 1 >= First); 1195 StringRef::iterator I = Position - 1; 1196 // We calculate the number of consecutive '\'s before the current position 1197 // by iterating backwards through our string. 1198 while (I >= First && *I == '\\') --I; 1199 // (Position - 1 - I) now contains the number of '\'s before the current 1200 // position. If it is odd, the character at 'Position' was escaped. 1201 return (Position - 1 - I) % 2 == 1; 1202 } 1203 1204 bool Scanner::scanFlowScalar(bool IsDoubleQuoted) { 1205 StringRef::iterator Start = Current; 1206 unsigned ColStart = Column; 1207 if (IsDoubleQuoted) { 1208 do { 1209 ++Current; 1210 while (Current != End && *Current != '"') 1211 ++Current; 1212 // Repeat until the previous character was not a '\' or was an escaped 1213 // backslash. 1214 } while (*(Current - 1) == '\\' && wasEscaped(Start + 1, Current)); 1215 } else { 1216 skip(1); 1217 while (true) { 1218 // Skip a ' followed by another '. 1219 if (Current + 1 < End && *Current == '\'' && *(Current + 1) == '\'') { 1220 skip(2); 1221 continue; 1222 } else if (*Current == '\'') 1223 break; 1224 StringRef::iterator i = skip_nb_char(Current); 1225 if (i == Current) { 1226 i = skip_b_break(Current); 1227 if (i == Current) 1228 break; 1229 Current = i; 1230 Column = 0; 1231 ++Line; 1232 } else { 1233 if (i == End) 1234 break; 1235 Current = i; 1236 ++Column; 1237 } 1238 } 1239 } 1240 skip(1); // Skip ending quote. 1241 Token T; 1242 T.Kind = Token::TK_Scalar; 1243 T.Range = StringRef(Start, Current - Start); 1244 TokenQueue.push_back(T); 1245 1246 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false); 1247 1248 IsSimpleKeyAllowed = false; 1249 1250 return true; 1251 } 1252 1253 bool Scanner::scanPlainScalar() { 1254 StringRef::iterator Start = Current; 1255 unsigned ColStart = Column; 1256 unsigned LeadingBlanks = 0; 1257 assert(Indent >= -1 && "Indent must be >= -1 !"); 1258 unsigned indent = static_cast<unsigned>(Indent + 1); 1259 while (true) { 1260 if (*Current == '#') 1261 break; 1262 1263 while (!isBlankOrBreak(Current)) { 1264 if ( FlowLevel && *Current == ':' 1265 && !(isBlankOrBreak(Current + 1) || *(Current + 1) == ',')) { 1266 setError("Found unexpected ':' while scanning a plain scalar", Current); 1267 return false; 1268 } 1269 1270 // Check for the end of the plain scalar. 1271 if ( (*Current == ':' && isBlankOrBreak(Current + 1)) 1272 || ( FlowLevel 1273 && (StringRef(Current, 1).find_first_of(",:?[]{}") 1274 != StringRef::npos))) 1275 break; 1276 1277 StringRef::iterator i = skip_nb_char(Current); 1278 if (i == Current) 1279 break; 1280 Current = i; 1281 ++Column; 1282 } 1283 1284 // Are we at the end? 1285 if (!isBlankOrBreak(Current)) 1286 break; 1287 1288 // Eat blanks. 1289 StringRef::iterator Tmp = Current; 1290 while (isBlankOrBreak(Tmp)) { 1291 StringRef::iterator i = skip_s_white(Tmp); 1292 if (i != Tmp) { 1293 if (LeadingBlanks && (Column < indent) && *Tmp == '\t') { 1294 setError("Found invalid tab character in indentation", Tmp); 1295 return false; 1296 } 1297 Tmp = i; 1298 ++Column; 1299 } else { 1300 i = skip_b_break(Tmp); 1301 if (!LeadingBlanks) 1302 LeadingBlanks = 1; 1303 Tmp = i; 1304 Column = 0; 1305 ++Line; 1306 } 1307 } 1308 1309 if (!FlowLevel && Column < indent) 1310 break; 1311 1312 Current = Tmp; 1313 } 1314 if (Start == Current) { 1315 setError("Got empty plain scalar", Start); 1316 return false; 1317 } 1318 Token T; 1319 T.Kind = Token::TK_Scalar; 1320 T.Range = StringRef(Start, Current - Start); 1321 TokenQueue.push_back(T); 1322 1323 // Plain scalars can be simple keys. 1324 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false); 1325 1326 IsSimpleKeyAllowed = false; 1327 1328 return true; 1329 } 1330 1331 bool Scanner::scanAliasOrAnchor(bool IsAlias) { 1332 StringRef::iterator Start = Current; 1333 unsigned ColStart = Column; 1334 skip(1); 1335 while(true) { 1336 if ( *Current == '[' || *Current == ']' 1337 || *Current == '{' || *Current == '}' 1338 || *Current == ',' 1339 || *Current == ':') 1340 break; 1341 StringRef::iterator i = skip_ns_char(Current); 1342 if (i == Current) 1343 break; 1344 Current = i; 1345 ++Column; 1346 } 1347 1348 if (Start == Current) { 1349 setError("Got empty alias or anchor", Start); 1350 return false; 1351 } 1352 1353 Token T; 1354 T.Kind = IsAlias ? Token::TK_Alias : Token::TK_Anchor; 1355 T.Range = StringRef(Start, Current - Start); 1356 TokenQueue.push_back(T); 1357 1358 // Alias and anchors can be simple keys. 1359 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false); 1360 1361 IsSimpleKeyAllowed = false; 1362 1363 return true; 1364 } 1365 1366 bool Scanner::scanBlockScalar(bool IsLiteral) { 1367 StringRef::iterator Start = Current; 1368 skip(1); // Eat | or > 1369 while(true) { 1370 StringRef::iterator i = skip_nb_char(Current); 1371 if (i == Current) { 1372 if (Column == 0) 1373 break; 1374 i = skip_b_break(Current); 1375 if (i != Current) { 1376 // We got a line break. 1377 Column = 0; 1378 ++Line; 1379 Current = i; 1380 continue; 1381 } else { 1382 // There was an error, which should already have been printed out. 1383 return false; 1384 } 1385 } 1386 Current = i; 1387 ++Column; 1388 } 1389 1390 if (Start == Current) { 1391 setError("Got empty block scalar", Start); 1392 return false; 1393 } 1394 1395 Token T; 1396 T.Kind = Token::TK_Scalar; 1397 T.Range = StringRef(Start, Current - Start); 1398 TokenQueue.push_back(T); 1399 return true; 1400 } 1401 1402 bool Scanner::scanTag() { 1403 StringRef::iterator Start = Current; 1404 unsigned ColStart = Column; 1405 skip(1); // Eat !. 1406 if (Current == End || isBlankOrBreak(Current)); // An empty tag. 1407 else if (*Current == '<') { 1408 skip(1); 1409 scan_ns_uri_char(); 1410 if (!consume('>')) 1411 return false; 1412 } else { 1413 // FIXME: Actually parse the c-ns-shorthand-tag rule. 1414 Current = skip_while(&Scanner::skip_ns_char, Current); 1415 } 1416 1417 Token T; 1418 T.Kind = Token::TK_Tag; 1419 T.Range = StringRef(Start, Current - Start); 1420 TokenQueue.push_back(T); 1421 1422 // Tags can be simple keys. 1423 saveSimpleKeyCandidate(TokenQueue.back(), ColStart, false); 1424 1425 IsSimpleKeyAllowed = false; 1426 1427 return true; 1428 } 1429 1430 bool Scanner::fetchMoreTokens() { 1431 if (IsStartOfStream) 1432 return scanStreamStart(); 1433 1434 scanToNextToken(); 1435 1436 if (Current == End) 1437 return scanStreamEnd(); 1438 1439 removeStaleSimpleKeyCandidates(); 1440 1441 unrollIndent(Column); 1442 1443 if (Column == 0 && *Current == '%') 1444 return scanDirective(); 1445 1446 if (Column == 0 && Current + 4 <= End 1447 && *Current == '-' 1448 && *(Current + 1) == '-' 1449 && *(Current + 2) == '-' 1450 && (Current + 3 == End || isBlankOrBreak(Current + 3))) 1451 return scanDocumentIndicator(true); 1452 1453 if (Column == 0 && Current + 4 <= End 1454 && *Current == '.' 1455 && *(Current + 1) == '.' 1456 && *(Current + 2) == '.' 1457 && (Current + 3 == End || isBlankOrBreak(Current + 3))) 1458 return scanDocumentIndicator(false); 1459 1460 if (*Current == '[') 1461 return scanFlowCollectionStart(true); 1462 1463 if (*Current == '{') 1464 return scanFlowCollectionStart(false); 1465 1466 if (*Current == ']') 1467 return scanFlowCollectionEnd(true); 1468 1469 if (*Current == '}') 1470 return scanFlowCollectionEnd(false); 1471 1472 if (*Current == ',') 1473 return scanFlowEntry(); 1474 1475 if (*Current == '-' && isBlankOrBreak(Current + 1)) 1476 return scanBlockEntry(); 1477 1478 if (*Current == '?' && (FlowLevel || isBlankOrBreak(Current + 1))) 1479 return scanKey(); 1480 1481 if (*Current == ':' && (FlowLevel || isBlankOrBreak(Current + 1))) 1482 return scanValue(); 1483 1484 if (*Current == '*') 1485 return scanAliasOrAnchor(true); 1486 1487 if (*Current == '&') 1488 return scanAliasOrAnchor(false); 1489 1490 if (*Current == '!') 1491 return scanTag(); 1492 1493 if (*Current == '|' && !FlowLevel) 1494 return scanBlockScalar(true); 1495 1496 if (*Current == '>' && !FlowLevel) 1497 return scanBlockScalar(false); 1498 1499 if (*Current == '\'') 1500 return scanFlowScalar(false); 1501 1502 if (*Current == '"') 1503 return scanFlowScalar(true); 1504 1505 // Get a plain scalar. 1506 StringRef FirstChar(Current, 1); 1507 if (!(isBlankOrBreak(Current) 1508 || FirstChar.find_first_of("-?:,[]{}#&*!|>'\"%@`") != StringRef::npos) 1509 || (*Current == '-' && !isBlankOrBreak(Current + 1)) 1510 || (!FlowLevel && (*Current == '?' || *Current == ':') 1511 && isBlankOrBreak(Current + 1)) 1512 || (!FlowLevel && *Current == ':' 1513 && Current + 2 < End 1514 && *(Current + 1) == ':' 1515 && !isBlankOrBreak(Current + 2))) 1516 return scanPlainScalar(); 1517 1518 setError("Unrecognized character while tokenizing."); 1519 return false; 1520 } 1521 1522 Stream::Stream(StringRef Input, SourceMgr &SM) 1523 : scanner(new Scanner(Input, SM)) 1524 , CurrentDoc(0) {} 1525 1526 Stream::~Stream() {} 1527 1528 bool Stream::failed() { return scanner->failed(); } 1529 1530 void Stream::printError(Node *N, const Twine &Msg) { 1531 SmallVector<SMRange, 1> Ranges; 1532 Ranges.push_back(N->getSourceRange()); 1533 scanner->printError( N->getSourceRange().Start 1534 , SourceMgr::DK_Error 1535 , Msg 1536 , Ranges); 1537 } 1538 1539 void Stream::handleYAMLDirective(const Token &t) { 1540 // TODO: Ensure version is 1.x. 1541 } 1542 1543 document_iterator Stream::begin() { 1544 if (CurrentDoc) 1545 report_fatal_error("Can only iterate over the stream once"); 1546 1547 // Skip Stream-Start. 1548 scanner->getNext(); 1549 1550 CurrentDoc.reset(new Document(*this)); 1551 return document_iterator(CurrentDoc); 1552 } 1553 1554 document_iterator Stream::end() { 1555 return document_iterator(); 1556 } 1557 1558 void Stream::skip() { 1559 for (document_iterator i = begin(), e = end(); i != e; ++i) 1560 i->skip(); 1561 } 1562 1563 Node::Node(unsigned int Type, OwningPtr<Document> &D, StringRef A) 1564 : Doc(D) 1565 , TypeID(Type) 1566 , Anchor(A) { 1567 SMLoc Start = SMLoc::getFromPointer(peekNext().Range.begin()); 1568 SourceRange = SMRange(Start, Start); 1569 } 1570 1571 Token &Node::peekNext() { 1572 return Doc->peekNext(); 1573 } 1574 1575 Token Node::getNext() { 1576 return Doc->getNext(); 1577 } 1578 1579 Node *Node::parseBlockNode() { 1580 return Doc->parseBlockNode(); 1581 } 1582 1583 BumpPtrAllocator &Node::getAllocator() { 1584 return Doc->NodeAllocator; 1585 } 1586 1587 void Node::setError(const Twine &Msg, Token &Tok) const { 1588 Doc->setError(Msg, Tok); 1589 } 1590 1591 bool Node::failed() const { 1592 return Doc->failed(); 1593 } 1594 1595 1596 1597 StringRef ScalarNode::getValue(SmallVectorImpl<char> &Storage) const { 1598 // TODO: Handle newlines properly. We need to remove leading whitespace. 1599 if (Value[0] == '"') { // Double quoted. 1600 // Pull off the leading and trailing "s. 1601 StringRef UnquotedValue = Value.substr(1, Value.size() - 2); 1602 // Search for characters that would require unescaping the value. 1603 StringRef::size_type i = UnquotedValue.find_first_of("\\\r\n"); 1604 if (i != StringRef::npos) 1605 return unescapeDoubleQuoted(UnquotedValue, i, Storage); 1606 return UnquotedValue; 1607 } else if (Value[0] == '\'') { // Single quoted. 1608 // Pull off the leading and trailing 's. 1609 StringRef UnquotedValue = Value.substr(1, Value.size() - 2); 1610 StringRef::size_type i = UnquotedValue.find('\''); 1611 if (i != StringRef::npos) { 1612 // We're going to need Storage. 1613 Storage.clear(); 1614 Storage.reserve(UnquotedValue.size()); 1615 for (; i != StringRef::npos; i = UnquotedValue.find('\'')) { 1616 StringRef Valid(UnquotedValue.begin(), i); 1617 Storage.insert(Storage.end(), Valid.begin(), Valid.end()); 1618 Storage.push_back('\''); 1619 UnquotedValue = UnquotedValue.substr(i + 2); 1620 } 1621 Storage.insert(Storage.end(), UnquotedValue.begin(), UnquotedValue.end()); 1622 return StringRef(Storage.begin(), Storage.size()); 1623 } 1624 return UnquotedValue; 1625 } 1626 // Plain or block. 1627 size_t trimtrail = Value.rfind(' '); 1628 return Value.drop_back( 1629 trimtrail == StringRef::npos ? 0 : Value.size() - trimtrail); 1630 } 1631 1632 StringRef ScalarNode::unescapeDoubleQuoted( StringRef UnquotedValue 1633 , StringRef::size_type i 1634 , SmallVectorImpl<char> &Storage) 1635 const { 1636 // Use Storage to build proper value. 1637 Storage.clear(); 1638 Storage.reserve(UnquotedValue.size()); 1639 for (; i != StringRef::npos; i = UnquotedValue.find_first_of("\\\r\n")) { 1640 // Insert all previous chars into Storage. 1641 StringRef Valid(UnquotedValue.begin(), i); 1642 Storage.insert(Storage.end(), Valid.begin(), Valid.end()); 1643 // Chop off inserted chars. 1644 UnquotedValue = UnquotedValue.substr(i); 1645 1646 assert(!UnquotedValue.empty() && "Can't be empty!"); 1647 1648 // Parse escape or line break. 1649 switch (UnquotedValue[0]) { 1650 case '\r': 1651 case '\n': 1652 Storage.push_back('\n'); 1653 if ( UnquotedValue.size() > 1 1654 && (UnquotedValue[1] == '\r' || UnquotedValue[1] == '\n')) 1655 UnquotedValue = UnquotedValue.substr(1); 1656 UnquotedValue = UnquotedValue.substr(1); 1657 break; 1658 default: 1659 if (UnquotedValue.size() == 1) 1660 // TODO: Report error. 1661 break; 1662 UnquotedValue = UnquotedValue.substr(1); 1663 switch (UnquotedValue[0]) { 1664 default: { 1665 Token T; 1666 T.Range = StringRef(UnquotedValue.begin(), 1); 1667 setError("Unrecognized escape code!", T); 1668 return ""; 1669 } 1670 case '\r': 1671 case '\n': 1672 // Remove the new line. 1673 if ( UnquotedValue.size() > 1 1674 && (UnquotedValue[1] == '\r' || UnquotedValue[1] == '\n')) 1675 UnquotedValue = UnquotedValue.substr(1); 1676 // If this was just a single byte newline, it will get skipped 1677 // below. 1678 break; 1679 case '0': 1680 Storage.push_back(0x00); 1681 break; 1682 case 'a': 1683 Storage.push_back(0x07); 1684 break; 1685 case 'b': 1686 Storage.push_back(0x08); 1687 break; 1688 case 't': 1689 case 0x09: 1690 Storage.push_back(0x09); 1691 break; 1692 case 'n': 1693 Storage.push_back(0x0A); 1694 break; 1695 case 'v': 1696 Storage.push_back(0x0B); 1697 break; 1698 case 'f': 1699 Storage.push_back(0x0C); 1700 break; 1701 case 'r': 1702 Storage.push_back(0x0D); 1703 break; 1704 case 'e': 1705 Storage.push_back(0x1B); 1706 break; 1707 case ' ': 1708 Storage.push_back(0x20); 1709 break; 1710 case '"': 1711 Storage.push_back(0x22); 1712 break; 1713 case '/': 1714 Storage.push_back(0x2F); 1715 break; 1716 case '\\': 1717 Storage.push_back(0x5C); 1718 break; 1719 case 'N': 1720 encodeUTF8(0x85, Storage); 1721 break; 1722 case '_': 1723 encodeUTF8(0xA0, Storage); 1724 break; 1725 case 'L': 1726 encodeUTF8(0x2028, Storage); 1727 break; 1728 case 'P': 1729 encodeUTF8(0x2029, Storage); 1730 break; 1731 case 'x': { 1732 if (UnquotedValue.size() < 3) 1733 // TODO: Report error. 1734 break; 1735 unsigned int UnicodeScalarValue; 1736 UnquotedValue.substr(1, 2).getAsInteger(16, UnicodeScalarValue); 1737 encodeUTF8(UnicodeScalarValue, Storage); 1738 UnquotedValue = UnquotedValue.substr(2); 1739 break; 1740 } 1741 case 'u': { 1742 if (UnquotedValue.size() < 5) 1743 // TODO: Report error. 1744 break; 1745 unsigned int UnicodeScalarValue; 1746 UnquotedValue.substr(1, 4).getAsInteger(16, UnicodeScalarValue); 1747 encodeUTF8(UnicodeScalarValue, Storage); 1748 UnquotedValue = UnquotedValue.substr(4); 1749 break; 1750 } 1751 case 'U': { 1752 if (UnquotedValue.size() < 9) 1753 // TODO: Report error. 1754 break; 1755 unsigned int UnicodeScalarValue; 1756 UnquotedValue.substr(1, 8).getAsInteger(16, UnicodeScalarValue); 1757 encodeUTF8(UnicodeScalarValue, Storage); 1758 UnquotedValue = UnquotedValue.substr(8); 1759 break; 1760 } 1761 } 1762 UnquotedValue = UnquotedValue.substr(1); 1763 } 1764 } 1765 Storage.insert(Storage.end(), UnquotedValue.begin(), UnquotedValue.end()); 1766 return StringRef(Storage.begin(), Storage.size()); 1767 } 1768 1769 Node *KeyValueNode::getKey() { 1770 if (Key) 1771 return Key; 1772 // Handle implicit null keys. 1773 { 1774 Token &t = peekNext(); 1775 if ( t.Kind == Token::TK_BlockEnd 1776 || t.Kind == Token::TK_Value 1777 || t.Kind == Token::TK_Error) { 1778 return Key = new (getAllocator()) NullNode(Doc); 1779 } 1780 if (t.Kind == Token::TK_Key) 1781 getNext(); // skip TK_Key. 1782 } 1783 1784 // Handle explicit null keys. 1785 Token &t = peekNext(); 1786 if (t.Kind == Token::TK_BlockEnd || t.Kind == Token::TK_Value) { 1787 return Key = new (getAllocator()) NullNode(Doc); 1788 } 1789 1790 // We've got a normal key. 1791 return Key = parseBlockNode(); 1792 } 1793 1794 Node *KeyValueNode::getValue() { 1795 if (Value) 1796 return Value; 1797 getKey()->skip(); 1798 if (failed()) 1799 return Value = new (getAllocator()) NullNode(Doc); 1800 1801 // Handle implicit null values. 1802 { 1803 Token &t = peekNext(); 1804 if ( t.Kind == Token::TK_BlockEnd 1805 || t.Kind == Token::TK_FlowMappingEnd 1806 || t.Kind == Token::TK_Key 1807 || t.Kind == Token::TK_FlowEntry 1808 || t.Kind == Token::TK_Error) { 1809 return Value = new (getAllocator()) NullNode(Doc); 1810 } 1811 1812 if (t.Kind != Token::TK_Value) { 1813 setError("Unexpected token in Key Value.", t); 1814 return Value = new (getAllocator()) NullNode(Doc); 1815 } 1816 getNext(); // skip TK_Value. 1817 } 1818 1819 // Handle explicit null values. 1820 Token &t = peekNext(); 1821 if (t.Kind == Token::TK_BlockEnd || t.Kind == Token::TK_Key) { 1822 return Value = new (getAllocator()) NullNode(Doc); 1823 } 1824 1825 // We got a normal value. 1826 return Value = parseBlockNode(); 1827 } 1828 1829 void MappingNode::increment() { 1830 if (failed()) { 1831 IsAtEnd = true; 1832 CurrentEntry = 0; 1833 return; 1834 } 1835 if (CurrentEntry) { 1836 CurrentEntry->skip(); 1837 if (Type == MT_Inline) { 1838 IsAtEnd = true; 1839 CurrentEntry = 0; 1840 return; 1841 } 1842 } 1843 Token T = peekNext(); 1844 if (T.Kind == Token::TK_Key || T.Kind == Token::TK_Scalar) { 1845 // KeyValueNode eats the TK_Key. That way it can detect null keys. 1846 CurrentEntry = new (getAllocator()) KeyValueNode(Doc); 1847 } else if (Type == MT_Block) { 1848 switch (T.Kind) { 1849 case Token::TK_BlockEnd: 1850 getNext(); 1851 IsAtEnd = true; 1852 CurrentEntry = 0; 1853 break; 1854 default: 1855 setError("Unexpected token. Expected Key or Block End", T); 1856 case Token::TK_Error: 1857 IsAtEnd = true; 1858 CurrentEntry = 0; 1859 } 1860 } else { 1861 switch (T.Kind) { 1862 case Token::TK_FlowEntry: 1863 // Eat the flow entry and recurse. 1864 getNext(); 1865 return increment(); 1866 case Token::TK_FlowMappingEnd: 1867 getNext(); 1868 case Token::TK_Error: 1869 // Set this to end iterator. 1870 IsAtEnd = true; 1871 CurrentEntry = 0; 1872 break; 1873 default: 1874 setError( "Unexpected token. Expected Key, Flow Entry, or Flow " 1875 "Mapping End." 1876 , T); 1877 IsAtEnd = true; 1878 CurrentEntry = 0; 1879 } 1880 } 1881 } 1882 1883 void SequenceNode::increment() { 1884 if (failed()) { 1885 IsAtEnd = true; 1886 CurrentEntry = 0; 1887 return; 1888 } 1889 if (CurrentEntry) 1890 CurrentEntry->skip(); 1891 Token T = peekNext(); 1892 if (SeqType == ST_Block) { 1893 switch (T.Kind) { 1894 case Token::TK_BlockEntry: 1895 getNext(); 1896 CurrentEntry = parseBlockNode(); 1897 if (CurrentEntry == 0) { // An error occurred. 1898 IsAtEnd = true; 1899 CurrentEntry = 0; 1900 } 1901 break; 1902 case Token::TK_BlockEnd: 1903 getNext(); 1904 IsAtEnd = true; 1905 CurrentEntry = 0; 1906 break; 1907 default: 1908 setError( "Unexpected token. Expected Block Entry or Block End." 1909 , T); 1910 case Token::TK_Error: 1911 IsAtEnd = true; 1912 CurrentEntry = 0; 1913 } 1914 } else if (SeqType == ST_Indentless) { 1915 switch (T.Kind) { 1916 case Token::TK_BlockEntry: 1917 getNext(); 1918 CurrentEntry = parseBlockNode(); 1919 if (CurrentEntry == 0) { // An error occurred. 1920 IsAtEnd = true; 1921 CurrentEntry = 0; 1922 } 1923 break; 1924 default: 1925 case Token::TK_Error: 1926 IsAtEnd = true; 1927 CurrentEntry = 0; 1928 } 1929 } else if (SeqType == ST_Flow) { 1930 switch (T.Kind) { 1931 case Token::TK_FlowEntry: 1932 // Eat the flow entry and recurse. 1933 getNext(); 1934 WasPreviousTokenFlowEntry = true; 1935 return increment(); 1936 case Token::TK_FlowSequenceEnd: 1937 getNext(); 1938 case Token::TK_Error: 1939 // Set this to end iterator. 1940 IsAtEnd = true; 1941 CurrentEntry = 0; 1942 break; 1943 case Token::TK_StreamEnd: 1944 case Token::TK_DocumentEnd: 1945 case Token::TK_DocumentStart: 1946 setError("Could not find closing ]!", T); 1947 // Set this to end iterator. 1948 IsAtEnd = true; 1949 CurrentEntry = 0; 1950 break; 1951 default: 1952 if (!WasPreviousTokenFlowEntry) { 1953 setError("Expected , between entries!", T); 1954 IsAtEnd = true; 1955 CurrentEntry = 0; 1956 break; 1957 } 1958 // Otherwise it must be a flow entry. 1959 CurrentEntry = parseBlockNode(); 1960 if (!CurrentEntry) { 1961 IsAtEnd = true; 1962 } 1963 WasPreviousTokenFlowEntry = false; 1964 break; 1965 } 1966 } 1967 } 1968 1969 Document::Document(Stream &S) : stream(S), Root(0) { 1970 if (parseDirectives()) 1971 expectToken(Token::TK_DocumentStart); 1972 Token &T = peekNext(); 1973 if (T.Kind == Token::TK_DocumentStart) 1974 getNext(); 1975 } 1976 1977 bool Document::skip() { 1978 if (stream.scanner->failed()) 1979 return false; 1980 if (!Root) 1981 getRoot(); 1982 Root->skip(); 1983 Token &T = peekNext(); 1984 if (T.Kind == Token::TK_StreamEnd) 1985 return false; 1986 if (T.Kind == Token::TK_DocumentEnd) { 1987 getNext(); 1988 return skip(); 1989 } 1990 return true; 1991 } 1992 1993 Token &Document::peekNext() { 1994 return stream.scanner->peekNext(); 1995 } 1996 1997 Token Document::getNext() { 1998 return stream.scanner->getNext(); 1999 } 2000 2001 void Document::setError(const Twine &Message, Token &Location) const { 2002 stream.scanner->setError(Message, Location.Range.begin()); 2003 } 2004 2005 bool Document::failed() const { 2006 return stream.scanner->failed(); 2007 } 2008 2009 Node *Document::parseBlockNode() { 2010 Token T = peekNext(); 2011 // Handle properties. 2012 Token AnchorInfo; 2013 parse_property: 2014 switch (T.Kind) { 2015 case Token::TK_Alias: 2016 getNext(); 2017 return new (NodeAllocator) AliasNode(stream.CurrentDoc, T.Range.substr(1)); 2018 case Token::TK_Anchor: 2019 if (AnchorInfo.Kind == Token::TK_Anchor) { 2020 setError("Already encountered an anchor for this node!", T); 2021 return 0; 2022 } 2023 AnchorInfo = getNext(); // Consume TK_Anchor. 2024 T = peekNext(); 2025 goto parse_property; 2026 case Token::TK_Tag: 2027 getNext(); // Skip TK_Tag. 2028 T = peekNext(); 2029 goto parse_property; 2030 default: 2031 break; 2032 } 2033 2034 switch (T.Kind) { 2035 case Token::TK_BlockEntry: 2036 // We got an unindented BlockEntry sequence. This is not terminated with 2037 // a BlockEnd. 2038 // Don't eat the TK_BlockEntry, SequenceNode needs it. 2039 return new (NodeAllocator) SequenceNode( stream.CurrentDoc 2040 , AnchorInfo.Range.substr(1) 2041 , SequenceNode::ST_Indentless); 2042 case Token::TK_BlockSequenceStart: 2043 getNext(); 2044 return new (NodeAllocator) 2045 SequenceNode( stream.CurrentDoc 2046 , AnchorInfo.Range.substr(1) 2047 , SequenceNode::ST_Block); 2048 case Token::TK_BlockMappingStart: 2049 getNext(); 2050 return new (NodeAllocator) 2051 MappingNode( stream.CurrentDoc 2052 , AnchorInfo.Range.substr(1) 2053 , MappingNode::MT_Block); 2054 case Token::TK_FlowSequenceStart: 2055 getNext(); 2056 return new (NodeAllocator) 2057 SequenceNode( stream.CurrentDoc 2058 , AnchorInfo.Range.substr(1) 2059 , SequenceNode::ST_Flow); 2060 case Token::TK_FlowMappingStart: 2061 getNext(); 2062 return new (NodeAllocator) 2063 MappingNode( stream.CurrentDoc 2064 , AnchorInfo.Range.substr(1) 2065 , MappingNode::MT_Flow); 2066 case Token::TK_Scalar: 2067 getNext(); 2068 return new (NodeAllocator) 2069 ScalarNode( stream.CurrentDoc 2070 , AnchorInfo.Range.substr(1) 2071 , T.Range); 2072 case Token::TK_Key: 2073 // Don't eat the TK_Key, KeyValueNode expects it. 2074 return new (NodeAllocator) 2075 MappingNode( stream.CurrentDoc 2076 , AnchorInfo.Range.substr(1) 2077 , MappingNode::MT_Inline); 2078 case Token::TK_DocumentStart: 2079 case Token::TK_DocumentEnd: 2080 case Token::TK_StreamEnd: 2081 default: 2082 // TODO: Properly handle tags. "[!!str ]" should resolve to !!str "", not 2083 // !!null null. 2084 return new (NodeAllocator) NullNode(stream.CurrentDoc); 2085 case Token::TK_Error: 2086 return 0; 2087 } 2088 llvm_unreachable("Control flow shouldn't reach here."); 2089 return 0; 2090 } 2091 2092 bool Document::parseDirectives() { 2093 bool isDirective = false; 2094 while (true) { 2095 Token T = peekNext(); 2096 if (T.Kind == Token::TK_TagDirective) { 2097 handleTagDirective(getNext()); 2098 isDirective = true; 2099 } else if (T.Kind == Token::TK_VersionDirective) { 2100 stream.handleYAMLDirective(getNext()); 2101 isDirective = true; 2102 } else 2103 break; 2104 } 2105 return isDirective; 2106 } 2107 2108 bool Document::expectToken(int TK) { 2109 Token T = getNext(); 2110 if (T.Kind != TK) { 2111 setError("Unexpected token", T); 2112 return false; 2113 } 2114 return true; 2115 } 2116 2117 OwningPtr<Document> document_iterator::NullDoc; 2118