Lines Matching refs:Tok
30 /// Re-lexes a sequence of tok::text tokens.
60 const Token &Tok = Toks[Pos.CurToken];
62 Pos.BufferStart = Tok.getText().begin();
63 Pos.BufferEnd = Tok.getText().end();
65 Pos.BufferStartLoc = Tok.getLocation();
100 if (P.Tok.is(tok::newline)) {
102 Token Newline = P.Tok;
104 if (P.Tok.isNot(tok::text)) {
110 if (P.Tok.isNot(tok::text)) {
115 Toks.push_back(P.Tok);
137 Result.setKind(tok::text);
154 bool lexWord(Token &Tok) {
183 formTokenWithChars(Tok, Loc, WordBegin, Length, Text);
187 bool lexDelimitedSeq(Token &Tok, char OpenDelim, char CloseDelim) {
228 formTokenWithChars(Tok, Loc, WordBegin,
312 assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
317 const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
319 Tok.is(tok::backslash_command) ? CMK_Backslash : CMK_At;
321 PC = S.actOnParamCommandStart(Tok.getLocation(),
322 Tok.getEndLocation(),
323 Tok.getCommandID(),
326 TPC = S.actOnTParamCommandStart(Tok.getLocation(),
327 Tok.getEndLocation(),
328 Tok.getCommandID(),
331 BC = S.actOnBlockCommandStart(Tok.getLocation(),
332 Tok.getEndLocation(),
333 Tok.getCommandID(),
374 else if (Tok.is(tok::newline)) {
375 Token PrevTok = Tok;
404 assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
406 const Token CommandTok = Tok;
434 assert(Tok.is(tok::html_start_tag));
436 S.actOnHTMLStartTagStart(Tok.getLocation(),
437 Tok.getHTMLTagStartName());
442 switch (Tok.getKind()) {
443 case tok::html_ident: {
444 Token Ident = Tok;
446 if (Tok.isNot(tok::html_equals)) {
451 Token Equals = Tok;
453 if (Tok.isNot(tok::html_quoted_string)) {
454 Diag(Tok.getLocation(),
459 while (Tok.is(tok::html_equals) ||
460 Tok.is(tok::html_quoted_string))
468 SourceRange(Tok.getLocation(),
469 Tok.getEndLocation()),
470 Tok.getHTMLQuotedString()));
475 case tok::html_greater:
478 Tok.getLocation(),
483 case tok::html_slash_greater:
486 Tok.getLocation(),
491 case tok::html_equals:
492 case tok::html_quoted_string:
493 Diag(Tok.getLocation(),
495 while (Tok.is(tok::html_equals) ||
496 Tok.is(tok::html_quoted_string))
498 if (Tok.is(tok::html_ident) ||
499 Tok.is(tok::html_greater) ||
500 Tok.is(tok::html_slash_greater))
521 Tok.getLocation(),
524 Diag(Tok.getLocation(),
528 Diag(Tok.getLocation(),
539 assert(Tok.is(tok::html_end_tag));
540 Token TokEndTag = Tok;
543 if (Tok.is(tok::html_greater)) {
544 Loc = Tok.getLocation();
557 switch (Tok.getKind()) {
558 case tok::verbatim_block_begin:
559 case tok::verbatim_line_name:
560 case tok::eof:
564 case tok::unknown_command:
565 Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
566 Tok.getEndLocation(),
567 Tok.getUnknownCommandName()));
571 case tok::backslash_command:
572 case tok::at_command: {
573 const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
580 Diag(Tok.getLocation(),
582 << Tok.is(tok::at_command)
584 << SourceRange(Tok.getLocation(), Tok.getEndLocation());
589 Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
590 Tok.getEndLocation(),
600 case tok::newline: {
602 if (Tok.is(tok::newline) || Tok.is(tok::eof)) {
606 // Also allow [tok::newline, tok::text, tok::newline] if the middle
607 // tok::text is just whitespace.
608 if (Tok.is(tok::text) && isWhitespace(Tok.getText())) {
609 Token WhitespaceTok = Tok;
611 if (Tok.is(tok::newline) || Tok.is(tok::eof)) {
615 // We have [tok::newline, tok::text, non-newline]. Put back tok::text.
624 case tok::html_start_tag:
628 case tok::html_end_tag:
632 case tok::text:
633 Content.push_back(S.actOnText(Tok.getLocation(),
634 Tok.getEndLocation(),
635 Tok.getText()));
639 case tok::verbatim_block_line:
640 case tok::verbatim_block_end:
641 case tok::verbatim_line_text:
642 case tok::html_ident:
643 case tok::html_equals:
644 case tok::html_quoted_string:
645 case tok::html_greater:
646 case tok::html_slash_greater:
656 assert(Tok.is(tok::verbatim_block_begin));
659 S.actOnVerbatimBlockStart(Tok.getLocation(),
660 Tok.getVerbatimBlockID());
665 if (Tok.is(tok::newline))
669 while (Tok.is(tok::verbatim_block_line) ||
670 Tok.is(tok::newline)) {
672 if (Tok.is(tok::verbatim_block_line)) {
673 Line = S.actOnVerbatimBlockLine(Tok.getLocation(),
674 Tok.getVerbatimBlockText());
676 if (Tok.is(tok::newline)) {
680 // Empty line, just a tok::newline.
681 Line = S.actOnVerbatimBlockLine(Tok.getLocation(), "");
687 if (Tok.is(tok::verbatim_block_end)) {
688 const CommandInfo *Info = Traits.getCommandInfo(Tok.getVerbatimBlockID());
689 S.actOnVerbatimBlockFinish(VB, Tok.getLocation(),
703 assert(Tok.is(tok::verbatim_line_name));
705 Token NameTok = Tok;
710 // Next token might not be a tok::verbatim_line_text if verbatim line
712 if (Tok.is(tok::verbatim_line_text)) {
713 TextBegin = Tok.getLocation();
714 Text = Tok.getVerbatimLineText();
729 switch (Tok.getKind()) {
730 case tok::text:
731 case tok::unknown_command:
732 case tok::backslash_command:
733 case tok::at_command:
734 case tok::html_start_tag:
735 case tok::html_end_tag:
738 case tok::verbatim_block_begin:
741 case tok::verbatim_line_name:
744 case tok::eof:
745 case tok::newline:
746 case tok::verbatim_block_line:
747 case tok::verbatim_block_end:
748 case tok::verbatim_line_text:
749 case tok::html_ident:
750 case tok::html_equals:
751 case tok::html_quoted_string:
752 case tok::html_greater:
753 case tok::html_slash_greater:
761 while (Tok.is(tok::newline))
765 while (Tok.isNot(tok::eof)) {
769 while (Tok.is(tok::newline))
