Lines Matching refs:Tok
36 bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const {
43 tok::ObjCKeywordKind Token::getObjCKeywordID() const {
45 return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword;
231 static size_t getSpellingSlow(const Token &Tok, const char *BufPtr,
233 assert(Tok.needsCleaning() && "getSpellingSlow called on simple token");
236 const char *BufEnd = BufPtr + Tok.getLength();
238 if (tok::isStringLiteral(Tok.getKind())) {
275 assert(Length < Tok.getLength() &&
326 std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr,
328 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
331 const char *TokStart = SourceMgr.getCharacterData(Tok.getLocation(),
339 if (!Tok.needsCleaning())
340 return std::string(TokStart, TokStart + Tok.getLength());
343 Result.resize(Tok.getLength());
344 Result.resize(getSpellingSlow(Tok, TokStart, LangOpts, &*Result.begin()));
351 /// Tok.getLength() bytes long. The actual length of the token is returned.
358 unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer,
361 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
365 if (Tok.is(tok::raw_identifier))
366 TokStart = Tok.getRawIdentifier().data();
367 else if (!Tok.hasUCN()) {
368 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
376 if (Tok.isLiteral())
377 TokStart = Tok.getLiteralData();
382 TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid);
392 if (!Tok.needsCleaning()) {
394 return Tok.getLength();
398 return getSpellingSlow(Tok, TokStart, LangOpts, const_cast<char*>(Buffer));
503 } while (TheTok.getKind() != tok::eof);
579 if (TheTok.getKind() == tok::eof) {
604 if (TheTok.getKind() == tok::comment) {
610 if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) {
620 if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) {
1157 tok::TokenKind TKind,
1182 Token Tok;
1183 lexer.LexFromRawLexer(Tok);
1184 if (Tok.isNot(TKind))
1186 SourceLocation TokenLoc = Tok.getLocation();
1192 Tok.getLength();
1209 return TokenLoc.getLocWithOffset(Tok.getLength() + NumWhitespaceChars);
1229 Token *Tok) {
1242 if (Tok) Tok->setFlag(Token::NeedsCleaning);
1245 if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode())
1259 return getCharAndSizeSlow(Ptr, Size, Tok);
1270 if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : nullptr)) {
1272 if (Tok) Tok->setFlag(Token::NeedsCleaning);
1493 FormTokenWithChars(Result, CurPtr, tok::raw_identifier);
1618 FormTokenWithChars(Result, CurPtr, tok::numeric_constant);
1717 tok::TokenKind Kind) {
1722 (Kind == tok::utf8_string_literal ||
1723 Kind == tok::utf16_string_literal ||
1724 Kind == tok::utf32_string_literal))
1740 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
1747 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
1775 tok::TokenKind Kind) {
1815 FormTokenWithChars(Result, CurPtr, tok::unknown);
1836 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
1869 FormTokenWithChars(Result, AfterLessPos, tok::less);
1883 FormTokenWithChars(Result, CurPtr, tok::angle_string_literal);
1892 tok::TokenKind Kind) {
1897 if (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant)
1901 else if (Kind == tok::utf8_char_constant)
1909 FormTokenWithChars(Result, CurPtr, tok::unknown);
1922 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
1929 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
1989 FormTokenWithChars(Result, CurPtr, tok::unknown);
2157 FormTokenWithChars(Result, CurPtr, tok::comment);
2173 Result.setKind(tok::comment);
2277 FormTokenWithChars(Result, CurPtr, tok::unknown);
2374 FormTokenWithChars(Result, CurPtr, tok::unknown);
2399 FormTokenWithChars(Result, CurPtr, tok::comment);
2461 if (Tmp.is(tok::code_completion)) {
2466 assert(Tmp.is(tok::eod) && "Unexpected token!");
2486 FormTokenWithChars(Result, CurPtr, tok::eod);
2499 FormTokenWithChars(Result, BufferEnd, tok::eof);
2544 /// the specified lexer will return a tok::l_paren token, 0 if it is something
2562 Token Tok;
2563 Lex(Tok);
2575 if (Tok.is(tok::eof))
2577 return Tok.is(tok::l_paren);
2857 FormTokenWithChars(Result, CurPtr, tok::unknown);
2922 FormTokenWithChars(Result, CurPtr, tok::unknown);
2935 tok::TokenKind Kind;
2947 FormTokenWithChars(Result, CurPtr, tok::code_completion);
2967 Kind = tok::unknown;
2986 Kind = tok::eod;
3048 tok::utf16_string_literal);
3053 tok::utf16_char_constant);
3061 tok::utf16_string_literal);
3071 tok::utf8_string_literal);
3076 tok::utf8_char_constant);
3087 tok::utf8_string_literal);
3106 tok::utf32_string_literal);
3111 tok::utf32_char_constant);
3119 tok::utf32_string_literal);
3135 tok::string_literal);
3149 tok::wide_string_literal);
3157 tok::wide_string_literal);
3162 tok::wide_char_constant);
3188 Kind = tok::unknown;
3195 return LexCharConstant(Result, CurPtr, tok::char_constant);
3201 return LexStringLiteral(Result, CurPtr, tok::string_literal);
3205 Kind = tok::question;
3208 Kind = tok::l_square;
3211 Kind = tok::r_square;
3214 Kind = tok::l_paren;
3217 Kind = tok::r_paren;
3220 Kind = tok::l_brace;
3223 Kind = tok::r_brace;
3233 Kind = tok::periodstar;
3237 Kind = tok::ellipsis;
3241 Kind = tok::period;
3247 Kind = tok::ampamp;
3250 Kind = tok::ampequal;
3253 Kind = tok::amp;
3258 Kind = tok::starequal;
3261 Kind = tok::star;
3268 Kind = tok::plusplus;
3271 Kind = tok::plusequal;
3273 Kind = tok::plus;
3280 Kind = tok::minusminus;
3285 Kind = tok::arrowstar;
3288 Kind = tok::arrow;
3291 Kind = tok::minusequal;
3293 Kind = tok::minus;
3297 Kind = tok::tilde;
3301 Kind = tok::exclaimequal;
3304 Kind = tok::exclaim;
3349 Kind = tok::slashequal;
3351 Kind = tok::slash;
3357 Kind = tok::percentequal;
3360 Kind = tok::r_brace; // '%>' -> '}'
3366 Kind = tok::hashhash; // '%:%:' -> '##'
3373 Kind = tok::hashat;
3382 Kind = tok::hash;
3385 Kind = tok::percent;
3395 Kind = tok::lesslessequal;
3407 Kind = tok::lesslessless;
3412 Kind = tok::lessless;
3416 Kind = tok::lessequal;
3428 Kind = tok::less;
3436 Kind = tok::l_square;
3439 Kind = tok::l_brace;
3441 Kind = tok::less;
3448 Kind = tok::greaterequal;
3454 Kind = tok::greatergreaterequal;
3463 Kind = tok::greatergreatergreater;
3468 Kind = tok::greatergreater;
3472 Kind = tok::greater;
3479 Kind = tok::caretequal;
3481 Kind = tok::caret;
3487 Kind = tok::pipeequal;
3493 Kind = tok::pipepipe;
3496 Kind = tok::pipe;
3502 Kind = tok::r_square; // ':>' -> ']'
3505 Kind = tok::coloncolon;
3508 Kind = tok::colon;
3512 Kind = tok::semi;
3521 Kind = tok::equalequal;
3524 Kind = tok::equal;
3528 Kind = tok::comma;
3533 Kind = tok::hashhash;
3536 Kind = tok::hashat;
3548 Kind = tok::hash;
3555 Kind = tok::at;
3557 Kind = tok::unknown;
3575 Kind = tok::unknown;
3580 Kind = tok::unknown;
3609 Kind = tok::unknown;
3636 FormTokenWithChars(Result, CurPtr, tok::hash);
3641 assert(Result.is(tok::eof) && "Preprocessor did not set tok:eof");
