Lines Matching refs:Token
1 //===--- TokenLexer.cpp - Lex from a token stream -------------------------===//
26 void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroArgs *Actuals) {
57 // creating separate source location entries for each token.
79 /// Create a TokenLexer for the specified token stream. This does not
80 /// take ownership of the specified token vector.
81 void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
99 // Set HasLeadingSpace/AtStartOfLine so that the first token will be
125 SmallVector<Token, 128> ResultToks;
132 // NextTokGetsSpace - When this is true, the next token appended to the
139 // preprocessor already verified that the following token is a macro name
141 const Token &CurTok = Tokens[i];
144 assert(ArgNo != -1 && "Token following # is not an argument?");
151 Token Res;
167 Res.setFlag(Token::LeadingSpace);
176 // Otherwise, if this is not an argument token, just add the token to the
185 ResultToks.back().setFlag(Token::LeadingSpace);
205 const Token *ResultArgToks;
209 const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
215 // If the arg token expanded into anything, append it.
224 Token &Tok = ResultToks[i];
236 // before the first token should match the whitespace of the arg
238 ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
244 // formal token, make sure the next token gets whitespace before it.
250 // Okay, we have a token that is either the LHS or RHS of a paste (##)
252 const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
257 // the expander trys to paste ',' with the first token of the __VA_ARG__
274 Token &Tok = ResultToks[i];
284 // If this token (the macro argument) was supposed to get leading
285 // whitespace, transfer this information onto the first token of the
289 // as in "A ## MACROARG". In valid code, the first token will get
296 ResultToks[ResultToks.size()-NumToks].setFlag(Token::LeadingSpace);
308 // Discard the argument token and skip (don't copy to the expansion
321 // If this is the __VA_ARGS__ token, and if the argument wasn't provided,
322 // and if the macro had at least one real argument, and if the token before
334 // then removal of the comma should produce a placemarker token (in C99
345 assert(!OwnsTokens && "This would leak if we already own the token list");
357 /// Lex - Lex and return a token from this macro stream.
359 void TokenLexer::Lex(Token &Tok) {
362 // If this is a macro (not a token stream), mark the macro enabled now
367 // token. This will delete "this" so remember the PP instance var.
372 // HandleEndOfTokenLexer may not return a token. If it doesn't, lex
379 // If this is the first token of the expanded result, we inherit spacing
383 // Get the next token to return.
388 // If this token is followed by a token paste (##) operator, paste the tokens!
389 // Note that ## is a normal token when not expanding a macro.
391 // When handling the microsoft /##/ extension, the final token is
392 // returned by PasteTokens, not the pasted token.
399 // The token's current location indicate where the token was lexed from. We
400 // need this information to compute the spelling of the token, but any
401 // diagnostics for the expanded token should appear as if they came from
404 if (ExpandLocStart.isValid() && // Don't do this for token streams.
405 // Check that the token's location was not already set properly.
420 // If this is the first token, set the lexical properties of the token to
423 Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
424 Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
429 // Change the kind of this identifier to the appropriate token kind, e.g.
445 // Otherwise, return a normal token.
451 /// If this returns true, the caller should immediately return the token.
452 bool TokenLexer::PasteTokens(Token &Tok) {
461 assert(!isAtEnd() && "No token on the RHS of a paste operator!");
463 // Get the RHS token.
464 const Token &RHS = Tokens[CurToken];
466 // Allocate space for the result token. This is guaranteed to be enough for
470 // Get the spelling of the LHS token in Buffer.
491 Token ResultTokTmp;
494 // Claim that the tmp token is a string_literal so that we can get the
501 // Lex the resultant pasted token into Result.
502 Token Result;
527 // Make a lexer to lex this string from. Lex just this one token.
533 // Lex a token in raw mode. This way it won't look up identifiers
534 // automatically, lexing off the end will return an eof token, and
535 // warnings are disabled. This returns true if the result token is the
539 // If we got an EOF token, we didn't form even ONE token. For example, we
543 // If pasting the two tokens didn't form a full new token, this is an
545 // unmodified and with RHS as the next token to lex.
557 // Explicitly convert the token location to have proper expansion
582 Result.setFlagValue(Token::StartOfLine , Tok.isAtStartOfLine());
583 Result.setFlagValue(Token::LeadingSpace, Tok.hasLeadingSpace());
592 // The token's current location indicate where the token was lexed from. We
593 // need this information to compute the spelling of the token, but any
594 // diagnostics for the expanded token should appear as if the token was
605 // Now that we got the result token, it will be subject to expansion. Since
606 // token pasting re-lexes the result token in raw mode, identifier information
609 // Look up the identifier info for the token. We disabled identifier lookup
616 /// isNextTokenLParen - If the next token lexed will pop this macro off the
617 /// expansion stack, return 2. If the next unexpanded token is a '(', return
636 /// first token on the next line.
637 void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok) {
643 assert(Macro && "Token streams can't paste comments");
656 "Not appropriate for token streams");
669 /// creates a single SLocEntry, and assigns SourceLocations to each token that
679 Token *&begin_tokens,
680 Token * end_tokens) {
693 // we can perform this "merge" since the token's spelling location depends
696 Token *NextTok = begin_tokens + 1;
700 break; // Token from different local/loaded location.
701 // Check that token is not before the previous token or more than 50
710 Token &LastConsecutiveTok = *(NextTok-1);
723 Token &Tok = *begin_tokens;
737 Token *begin_tokens,
738 Token *end_tokens) {
745 // If there's only one token just create a SLocEntry for it.
747 Token &Tok = *begin_tokens;
