Lines Matching full:tokenizer
55 /// \brief Simple tokenizer for the parser.
236 const TokenInfo NameToken = Tokenizer->consumeNextToken();
238 const TokenInfo OpenToken = Tokenizer->consumeNextToken();
247 while (Tokenizer->nextTokenKind() != TokenInfo::TK_Eof) {
248 if (Tokenizer->nextTokenKind() == TokenInfo::TK_CloseParen) {
250 EndToken = Tokenizer->consumeNextToken();
255 const TokenInfo CommaToken = Tokenizer->consumeNextToken();
266 ArgValue.Text = Tokenizer->peekNextToken().Text;
267 ArgValue.Range = Tokenizer->peekNextToken().Range;
279 if (Tokenizer->peekNextToken().Kind == TokenInfo::TK_Period) {
281 Tokenizer->consumeNextToken(); // consume the period.
282 const TokenInfo BindToken = Tokenizer->consumeNextToken();
283 const TokenInfo OpenToken = Tokenizer->consumeNextToken();
284 const TokenInfo IDToken = Tokenizer->consumeNextToken();
285 const TokenInfo CloseToken = Tokenizer->consumeNextToken();
324 switch (Tokenizer->nextTokenKind()) {
326 *Value = Tokenizer->consumeNextToken().Value;
333 Error->addError(Tokenizer->consumeNextToken().Range,
338 // This error was already reported by the tokenizer.
346 const TokenInfo Token = Tokenizer->consumeNextToken();
354 Parser::Parser(CodeTokenizer *Tokenizer, Sema *S,
356 : Tokenizer(Tokenizer), S(S), Error(Error) {}
383 CodeTokenizer Tokenizer(Code, Error);
384 if (!Parser(&Tokenizer, S, Error).parseExpressionImpl(Value)) return false;
385 if (Tokenizer.peekNextToken().Kind != TokenInfo::TK_Eof) {
386 Error->addError(Tokenizer.peekNextToken().Range,
