Lines Matching refs:token
26 #include "Token.h"
49 static DirectiveType getDirective(const pp::Token *token)
65 if (token->type != pp::Token::IDENTIFIER)
68 if (token->text == kDirectiveDefine)
70 else if (token->text == kDirectiveUndef)
72 else if (token->text == kDirectiveIf)
74 else if (token->text == kDirectiveIfdef)
76 else if (token->text == kDirectiveIfndef)
78 else if (token->text == kDirectiveElse)
80 else if (token->text == kDirectiveElif)
82 else if (token->text == kDirectiveEndif)
84 else if (token->text == kDirectiveError)
86 else if (token->text == kDirectivePragma)
88 else if (token->text == kDirectiveExtension)
90 else if (token->text == kDirectiveVersion)
92 else if (token->text == kDirectiveLine)
114 // Returns true if the token represents End Of Directive.
115 static bool isEOD(const pp::Token *token)
117 return (token->type == '\n') || (token->type == pp::Token::LAST);
120 static void skipUntilEOD(pp::Lexer *lexer, pp::Token *token)
122 while(!isEOD(token))
124 lexer->lex(token);
158 void lex(Token *token) override
162 mLexer->lex(token);
163 if (token->type != Token::IDENTIFIER)
165 if (token->text != kDefined)
169 mLexer->lex(token);
170 if (token->type == '(')
173 mLexer->lex(token);
176 if (token->type != Token::IDENTIFIER)
178 mDiagnostics->report(Diagnostics::PP_UNEXPECTED_TOKEN, token->location, token->text);
179 skipUntilEOD(mLexer, token);
182 MacroSet::const_iterator iter = mMacroSet->find(token->text);
187 mLexer->lex(token);
188 if (token->type != ')')
191 token->location, token->text);
192 skipUntilEOD(mLexer, token);
198 // Convert the current token into a CONST_INT token.
199 token->type = Token::CONST_INT;
200 token->text = expression;
229 void DirectiveParser::lex(Token *token)
233 mTokenizer->lex(token);
235 if (token->type == Token::PP_HASH)
237 parseDirective(token);
240 else if (!isEOD(token))
245 if (token->type == Token::LAST)
256 } while (skipping() || (token->type == '\n'));
261 void DirectiveParser::parseDirective(Token *token)
263 assert(token->type == Token::PP_HASH);
265 mTokenizer->lex(token);
266 if (isEOD(token))
272 DirectiveType directive = getDirective(token);
278 skipUntilEOD(mTokenizer, token);
286 token->location, token->text);
287 skipUntilEOD(mTokenizer, token);
290 parseDefine(token);
293 parseUndef(token);
296 parseIf(token);
299 parseIfdef(token);
302 parseIfndef(token);
305 parseElse(token);
308 parseElif(token);
311 parseEndif(token);
314 parseError(token);
317 parsePragma(token);
320 parseExtension(token);
323 parseVersion(token);
326 parseLine(token);
333 skipUntilEOD(mTokenizer, token);
334 if (token->type == Token::LAST)
337 token->location, token->text);
341 void DirectiveParser::parseDefine(Token *token)
343 assert(getDirective(token) == DIRECTIVE_DEFINE);
345 mTokenizer->lex(token);
346 if (token->type != Token::IDENTIFIER)
349 token->location, token->text);
352 if (isMacroPredefined(token->text, *mMacroSet))
355 token->location, token->text);
358 if (isMacroNameReserved(token->text))
361 token->location, token->text);
369 if (hasDoubleUnderscores(token->text))
371 mDiagnostics->report(Diagnostics::PP_WARNING_MACRO_NAME_RESERVED, token->location,
372 token->text);
377 macro->name = token->text;
379 mTokenizer->lex(token);
380 if (token->type == '(' && !token->hasLeadingSpace())
385 mTokenizer->lex(token);
386 if (token->type != Token::IDENTIFIER)
389 if (std::find(macro->parameters.begin(), macro->parameters.end(), token->text) != macro->parameters.end())
392 token->location, token->text);
396 macro->parameters.push_back(token->text);
398 mTokenizer->lex(token); // Get ','.
399 } while (token->type == ',');
401 if (token->type != ')')
404 token->location,
405 token->text);
408 mTokenizer->lex(token); // Get ')'.
411 while ((token->type != '\n') && (token->type != Token::LAST))
413 // Reset the token location because it is unnecessary in replacement
414 // list. Resetting it also allows us to reuse Token::equals() to
416 token->location = SourceLocation();
417 macro->replacements.push_back(*token);
418 mTokenizer->lex(token);
431 mDiagnostics->report(Diagnostics::PP_MACRO_REDEFINED, token->location, macro->name);
437 void DirectiveParser::parseUndef(Token *token)
439 assert(getDirective(token) == DIRECTIVE_UNDEF);
441 mTokenizer->lex(token);
442 if (token->type != Token::IDENTIFIER)
445 token->location, token->text);
449 MacroSet::iterator iter = mMacroSet->find(token->text);
455 token->location, token->text);
461 token->location, token->text);
470 mTokenizer->lex(token);
471 if (!isEOD(token))
473 mDiagnostics->report(Diagnostics::PP_UNEXPECTED_TOKEN, token->location, token->text);
474 skipUntilEOD(mTokenizer, token);
478 void DirectiveParser::parseIf(Token *token)
480 assert(getDirective(token) == DIRECTIVE_IF);
481 parseConditionalIf(token);
484 void DirectiveParser::parseIfdef(Token *token)
486 assert(getDirective(token) == DIRECTIVE_IFDEF);
487 parseConditionalIf(token);
490 void DirectiveParser::parseIfndef(Token *token)
492 assert(getDirective(token) == DIRECTIVE_IFNDEF);
493 parseConditionalIf(token);
496 void DirectiveParser::parseElse(Token *token)
498 assert(getDirective(token) == DIRECTIVE_ELSE);
503 token->location, token->text);
504 skipUntilEOD(mTokenizer, token);
512 skipUntilEOD(mTokenizer, token);
518 token->location, token->text);
519 skipUntilEOD(mTokenizer, token);
528 mTokenizer->lex(token);
529 if (!isEOD(token))
532 token->location, token->text);
533 skipUntilEOD(mTokenizer, token);
537 void DirectiveParser::parseElif(Token *token)
539 assert(getDirective(token) == DIRECTIVE_ELIF);
544 token->location, token->text);
545 skipUntilEOD(mTokenizer, token);
553 skipUntilEOD(mTokenizer, token);
559 token->location, token->text);
560 skipUntilEOD(mTokenizer, token);
568 skipUntilEOD(mTokenizer, token);
572 int expression = parseExpressionIf(token);
577 void DirectiveParser::parseEndif(Token *token)
579 assert(getDirective(token) == DIRECTIVE_ENDIF);
584 token->location, token->text);
585 skipUntilEOD(mTokenizer, token);
592 mTokenizer->lex(token);
593 if (!isEOD(token))
596 token->location, token->text);
597 skipUntilEOD(mTokenizer, token);
601 void DirectiveParser::parseError(Token *token)
603 assert(getDirective(token) == DIRECTIVE_ERROR);
606 mTokenizer->lex(token);
607 while ((token->type != '\n') && (token->type != Token::LAST))
609 stream << *token;
610 mTokenizer->lex(token);
612 mDirectiveHandler->handleError(token->location, stream.str());
616 void DirectiveParser::parsePragma(Token *token)
618 assert(getDirective(token) == DIRECTIVE_PRAGMA);
632 mTokenizer->lex(token);
633 bool stdgl = token->text == "STDGL";
636 mTokenizer->lex(token);
638 while ((token->type != '\n') && (token->type != Token::LAST))
643 name = token->text;
644 valid = valid && (token->type == Token::IDENTIFIER);
647 valid = valid && (token->type == '(');
650 value = token->text;
651 valid = valid && (token->type == Token::IDENTIFIER);
654 valid = valid && (token->type == ')');
660 mTokenizer->lex(token);
668 mDiagnostics->report(Diagnostics::PP_UNRECOGNIZED_PRAGMA, token->location, name);
672 mDirectiveHandler->handlePragma(token->location, name, value, stdgl);
676 void DirectiveParser::parseExtension(Token *token)
678 assert(getDirective(token) == DIRECTIVE_EXTENSION);
691 mTokenizer->lex(token);
692 while ((token->type != '\n') && (token->type != Token::LAST))
697 if (valid && (token->type != Token::IDENTIFIER))
699 mDiagnostics->report(Diagnostics::PP_INVALID_EXTENSION_NAME, token->location,
700 token->text);
704 name = token->text;
707 if (valid && (token->type != ':'))
709 mDiagnostics->report(Diagnostics::PP_UNEXPECTED_TOKEN, token->location,
710 token->text);
715 if (valid && (token->type != Token::IDENTIFIER))
718 token->location, token->text);
722 behavior = token->text;
727 mDiagnostics->report(Diagnostics::PP_UNEXPECTED_TOKEN, token->location,
728 token->text);
733 mTokenizer->lex(token);
737 mDiagnostics->report(Diagnostics::PP_INVALID_EXTENSION_DIRECTIVE, token->location,
738 token->text);
746 token->location, token->text);
752 token->location, token->text);
756 mDirectiveHandler->handleExtension(token->location, name, behavior);
759 void DirectiveParser::parseVersion(Token *token)
761 assert(getDirective(token) == DIRECTIVE_VERSION);
765 mDiagnostics->report(Diagnostics::PP_VERSION_NOT_FIRST_STATEMENT, token->location,
766 token->text);
767 skipUntilEOD(mTokenizer, token);
782 mTokenizer->lex(token);
783 while (valid && (token->type != '\n') && (token->type != Token::LAST))
788 if (token->type != Token::CONST_INT)
791 token->location, token->text);
794 if (valid && !token->iValue(&version))
797 token->location, token->text);
806 if (token->type != Token::IDENTIFIER || token->text != "es")
809 token->location, token->text);
816 token->location, token->text);
821 mTokenizer->lex(token);
827 token->location, token->text);
831 if (valid && version >= 300 && token->location.line > 1)
833 mDiagnostics->report(Diagnostics::PP_VERSION_NOT_FIRST_LINE_ESSL3, token->location,
834 token->text);
840 mDirectiveHandler->handleVersion(token->location, version);
846 void DirectiveParser::parseLine(Token *token)
848 assert(getDirective(token) == DIRECTIVE_LINE);
856 // Lex the first token after "#line" so we can check it for EOD.
857 macroExpander.lex(token);
859 if (isEOD(token))
861 mDiagnostics->report(Diagnostics::PP_INVALID_LINE_DIRECTIVE, token->location, token->text);
873 // The first token was lexed earlier to check if it was EOD. Include
874 // the token in parsing for a second time by setting the
876 expressionParser.parse(token, &line, true, errorSettings, &valid);
877 if (!isEOD(token) && valid)
881 // advanced to the first token of the file expression - this is the
882 // token that makes the parser reduce the "input" rule for the line
885 expressionParser.parse(token, &file, true, errorSettings, &valid);
888 if (!isEOD(token))
893 token->location, token->text);
896 skipUntilEOD(mTokenizer, token);
917 void DirectiveParser::parseConditionalIf(Token *token)
920 block.type = token->text;
921 block.location = token->location;
929 skipUntilEOD(mTokenizer, token);
934 DirectiveType directive = getDirective(token);
940 expression = parseExpressionIf(token);
943 expression = parseExpressionIfdef(token);
946 expression = parseExpressionIfdef(token) == 0 ? 1 : 0;
958 int DirectiveParser::parseExpressionIf(Token *token)
960 assert((getDirective(token) == DIRECTIVE_IF) || (getDirective(token) == DIRECTIVE_ELIF));
972 expressionParser.parse(token, &expression, false, errorSettings, &valid);
975 if (!isEOD(token))
978 token->location, token->text);
979 skipUntilEOD(mTokenizer, token);
985 int DirectiveParser::parseExpressionIfdef(Token* token)
987 assert((getDirective(token) == DIRECTIVE_IFDEF) ||
988 (getDirective(token) == DIRECTIVE_IFNDEF));
990 mTokenizer->lex(token);
991 if (token->type != Token::IDENTIFIER)
994 token->location, token->text);
995 skipUntilEOD(mTokenizer, token);
999 MacroSet::const_iterator iter = mMacroSet->find(token->text);
1003 mTokenizer->lex(token);
1004 if (!isEOD(token))
1007 token->location, token->text);
1008 skipUntilEOD(mTokenizer, token);
