Lines Matching defs:Tok
46 bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const {
53 tok::ObjCKeywordKind Token::getObjCKeywordID() const {
55 return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword;
289 std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr,
291 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
295 const char* TokStart = SourceMgr.getCharacterData(Tok.getLocation(),
302 if (!Tok.needsCleaning())
303 return std::string(TokStart, TokStart+Tok.getLength());
306 Result.reserve(Tok.getLength());
309 for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
315 assert(Result.size() != unsigned(Tok.getLength()) &&
323 /// Tok.getLength() bytes long. The actual length of the token is returned.
330 unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer,
333 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
337 if (Tok.is(tok::raw_identifier))
338 TokStart = Tok.getRawIdentifierData();
339 else if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
346 if (Tok.isLiteral())
347 TokStart = Tok.getLiteralData();
352 TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid);
362 if (!Tok.needsCleaning()) {
364 return Tok.getLength();
369 for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
375 assert(unsigned(OutBuf-Buffer) != Tok.getLength() &&
475 } while (TheTok.getKind() != tok::eof);
546 if (TheTok.getKind() == tok::eof) {
572 if (TheTok.getKind() == tok::comment)
575 if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) {
584 if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) {
1277 tok::TokenKind TKind,
1302 Token Tok;
1303 lexer.LexFromRawLexer(Tok);
1304 if (Tok.isNot(TKind))
1306 SourceLocation TokenLoc = Tok.getLocation();
1312 Tok.getLength();
1322 return TokenLoc.getLocWithOffset(Tok.getLength() + NumWhitespaceChars);
1343 Token *Tok) {
1356 if (Tok) Tok->setFlag(Token::NeedsCleaning);
1359 if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode())
1373 return getCharAndSizeSlow(Ptr, Size, Tok);
1384 if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : 0)) {
1386 if (Tok) Tok->setFlag(Token::NeedsCleaning);
1484 FormTokenWithChars(Result, CurPtr, tok::raw_identifier);
1572 FormTokenWithChars(Result, CurPtr, tok::numeric_constant);
1619 tok::TokenKind Kind) {
1623 (Kind == tok::utf8_string_literal ||
1624 Kind == tok::utf16_string_literal ||
1625 Kind == tok::utf32_string_literal))
1639 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
1646 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
1672 tok::TokenKind Kind) {
1712 FormTokenWithChars(Result, CurPtr, tok::unknown);
1733 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
1764 FormTokenWithChars(Result, AfterLessPos, tok::less);
1778 FormTokenWithChars(Result, CurPtr, tok::angle_string_literal);
1786 tok::TokenKind Kind) {
1790 (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant))
1797 FormTokenWithChars(Result, CurPtr, tok::unknown);
1811 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
1816 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
1877 FormTokenWithChars(Result, CurPtr, tok::unknown);
2023 FormTokenWithChars(Result, CurPtr, tok::comment);
2039 Result.setKind(tok::comment);
2142 FormTokenWithChars(Result, CurPtr, tok::unknown);
2238 FormTokenWithChars(Result, CurPtr, tok::unknown);
2263 FormTokenWithChars(Result, CurPtr, tok::comment);
2325 if (Tmp.is(tok::code_completion)) {
2330 assert(Tmp.is(tok::eod) && "Unexpected token!");
2350 FormTokenWithChars(Result, CurPtr, tok::eod);
2362 FormTokenWithChars(Result, BufferEnd, tok::eof);
2390 /// the specified lexer will return a tok::l_paren token, 0 if it is something
2405 Token Tok;
2406 Tok.startToken();
2407 LexTokenInternal(Tok);
2416 if (Tok.is(tok::eof))
2418 return Tok.is(tok::l_paren);
2561 FormTokenWithChars(Result, CurPtr, tok::unknown);
2573 tok::TokenKind Kind;
2592 FormTokenWithChars(Result, CurPtr, tok::code_completion);
2616 Kind = tok::unknown;
2633 Kind = tok::eod;
2690 tok::utf16_string_literal);
2695 tok::utf16_char_constant);
2702 tok::utf16_string_literal);
2712 tok::utf8_string_literal);
2723 tok::utf8_string_literal);
2742 tok::utf32_string_literal);
2747 tok::utf32_char_constant);
2754 tok::utf32_string_literal);
2770 tok::string_literal);
2784 tok::wide_string_literal);
2792 tok::wide_string_literal);
2797 tok::wide_char_constant);
2823 Kind = tok::unknown;
2830 return LexCharConstant(Result, CurPtr, tok::char_constant);
2836 return LexStringLiteral(Result, CurPtr, tok::string_literal);
2840 Kind = tok::question;
2843 Kind = tok::l_square;
2846 Kind = tok::r_square;
2849 Kind = tok::l_paren;
2852 Kind = tok::r_paren;
2855 Kind = tok::l_brace;
2858 Kind = tok::r_brace;
2868 Kind = tok::periodstar;
2872 Kind = tok::ellipsis;
2876 Kind = tok::period;
2882 Kind = tok::ampamp;
2885 Kind = tok::ampequal;
2888 Kind = tok::amp;
2893 Kind = tok::starequal;
2896 Kind = tok::star;
2903 Kind = tok::plusplus;
2906 Kind = tok::plusequal;
2908 Kind = tok::plus;
2915 Kind = tok::minusminus;
2920 Kind = tok::arrowstar;
2923 Kind = tok::arrow;
2926 Kind = tok::minusequal;
2928 Kind = tok::minus;
2932 Kind = tok::tilde;
2936 Kind = tok::exclaimequal;
2939 Kind = tok::exclaim;
2975 Kind = tok::slashequal;
2977 Kind = tok::slash;
2983 Kind = tok::percentequal;
2986 Kind = tok::r_brace; // '%>' -> '}'
2992 Kind = tok::hashhash; // '%:%:' -> '##'
2999 Kind = tok::hashat;
3006 FormTokenWithChars(Result, CurPtr, tok::hash);
3026 Kind = tok::hash;
3029 Kind = tok::percent;
3039 Kind = tok::lesslessequal;
3051 Kind = tok::lesslessless;
3056 Kind = tok::lessless;
3060 Kind = tok::lessequal;
3072 Kind = tok::less;
3080 Kind = tok::l_square;
3083 Kind = tok::l_brace;
3085 Kind = tok::less;
3092 Kind = tok::greaterequal;
3098 tok::greatergreaterequal;
3107 Kind = tok::greatergreatergreater;
3112 Kind = tok::greatergreater;
3116 Kind = tok::greater;
3123 Kind = tok::caretequal;
3125 Kind = tok::caret;
3131 Kind = tok::pipeequal;
3137 Kind = tok::pipepipe;
3140 Kind = tok::pipe;
3146 Kind = tok::r_square; // ':>' -> ']'
3149 Kind = tok::coloncolon;
3152 Kind = tok::colon;
3156 Kind = tok::semi;
3165 Kind = tok::equalequal;
3168 Kind = tok::equal;
3172 Kind = tok::comma;
3177 Kind = tok::hashhash;
3180 Kind = tok::hashat;
3190 FormTokenWithChars(Result, CurPtr, tok::hash);
3209 Kind = tok::hash;
3216 Kind = tok::at;
3218 Kind = tok::unknown;
3225 Kind = tok::unknown;
