1 /* 2 // 3 // Copyright (c) 2002-2014 The ANGLE Project Authors. All rights reserved. 4 // Use of this source code is governed by a BSD-style license that can be 5 // found in the LICENSE file. 6 // 7 8 This file contains the Lex specification for GLSL ES preprocessor. 9 Based on Microsoft Visual Studio 2010 Preprocessor Grammar: 10 http://msdn.microsoft.com/en-us/library/2scxys89.aspx 11 12 IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh. 13 */ 14 15 %top{ 16 // 17 // Copyright (c) 2011-2014 The ANGLE Project Authors. All rights reserved. 18 // Use of this source code is governed by a BSD-style license that can be 19 // found in the LICENSE file. 20 // 21 22 // This file is auto-generated by generate_parser.sh. DO NOT EDIT! 23 } 24 25 %{ 26 #include "Tokenizer.h" 27 28 #include "DiagnosticsBase.h" 29 #include "Token.h" 30 31 #if defined(__GNUC__) 32 // Triggered by the auto-generated yy_fatal_error function. 33 #pragma GCC diagnostic ignored "-Wmissing-noreturn" 34 #endif 35 36 typedef std::string YYSTYPE; 37 typedef pp::SourceLocation YYLTYPE; 38 39 // Use the unused yycolumn variable to track file (string) number. 40 #define yyfileno yycolumn 41 42 #define YY_USER_INIT \ 43 do { \ 44 yyfileno = 0; \ 45 yylineno = 1; \ 46 yyextra->leadingSpace = false; \ 47 yyextra->lineStart = true; \ 48 } while(0); 49 50 #define YY_USER_ACTION \ 51 do \ 52 { \ 53 pp::Input* input = &yyextra->input; \ 54 pp::Input::Location* scanLoc = &yyextra->scanLoc; \ 55 while ((scanLoc->sIndex < input->count()) && \ 56 (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \ 57 { \ 58 scanLoc->cIndex -= input->length(scanLoc->sIndex++); \ 59 ++yyfileno; yylineno = 1; \ 60 } \ 61 yylloc->file = yyfileno; \ 62 yylloc->line = yylineno; \ 63 scanLoc->cIndex += yyleng; \ 64 } while(0); 65 66 #define YY_INPUT(buf, result, maxSize) \ 67 result = yyextra->input.read(buf, maxSize); 68 69 %} 70 71 %option noyywrap nounput never-interactive 72 %option reentrant bison-bridge bison-locations 73 %option prefix="pp" 74 %option extra-type="pp::Tokenizer::Context*" 75 %x COMMENT 76 77 NEWLINE \n|\r|\r\n 78 IDENTIFIER [_a-zA-Z][_a-zA-Z0-9]* 79 PUNCTUATOR [][<>(){}.+-/*%^|&~=!:;,?] 80 81 DECIMAL_CONSTANT [1-9][0-9]*[uU]? 82 OCTAL_CONSTANT 0[0-7]*[uU]? 83 HEXADECIMAL_CONSTANT 0[xX][0-9a-fA-F]+[uU]? 84 85 DIGIT [0-9] 86 EXPONENT_PART [eE][+-]?{DIGIT}+ 87 FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".") 88 89 %% 90 91 /* Line comment */ 92 "//"[^\r\n]* 93 94 /* Block comment */ 95 /* Line breaks are just counted - not returned. */ 96 /* The comment is replaced by a single space. */ 97 "/*" { BEGIN(COMMENT); } 98 <COMMENT>[^*\r\n]+ 99 <COMMENT>"*" 100 <COMMENT>{NEWLINE} { ++yylineno; } 101 <COMMENT>"*/" { 102 yyextra->leadingSpace = true; 103 BEGIN(INITIAL); 104 } 105 106 # { 107 // # is only valid at start of line for preprocessor directives. 108 yylval->assign(1, yytext[0]); 109 return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER; 110 } 111 112 {IDENTIFIER} { 113 yylval->assign(yytext, yyleng); 114 return pp::Token::IDENTIFIER; 115 } 116 117 ({DECIMAL_CONSTANT}[uU]?)|({OCTAL_CONSTANT}[uU]?)|({HEXADECIMAL_CONSTANT}[uU]?) { 118 yylval->assign(yytext, yyleng); 119 return pp::Token::CONST_INT; 120 } 121 122 ({DIGIT}+{EXPONENT_PART}[fF]?)|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?[fF]?) { 123 yylval->assign(yytext, yyleng); 124 return pp::Token::CONST_FLOAT; 125 } 126 127 /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */ 128 /* Rule to catch all invalid integers and floats. */ 129 ({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) { 130 yylval->assign(yytext, yyleng); 131 return pp::Token::PP_NUMBER; 132 } 133 134 "++" { 135 yylval->assign(yytext, yyleng); 136 return pp::Token::OP_INC; 137 } 138 "--" { 139 yylval->assign(yytext, yyleng); 140 return pp::Token::OP_DEC; 141 } 142 "<<" { 143 yylval->assign(yytext, yyleng); 144 return pp::Token::OP_LEFT; 145 } 146 ">>" { 147 yylval->assign(yytext, yyleng); 148 return pp::Token::OP_RIGHT; 149 } 150 "<=" { 151 yylval->assign(yytext, yyleng); 152 return pp::Token::OP_LE; 153 } 154 ">=" { 155 yylval->assign(yytext, yyleng); 156 return pp::Token::OP_GE; 157 } 158 "==" { 159 yylval->assign(yytext, yyleng); 160 return pp::Token::OP_EQ; 161 } 162 "!=" { 163 yylval->assign(yytext, yyleng); 164 return pp::Token::OP_NE; 165 } 166 "&&" { 167 yylval->assign(yytext, yyleng); 168 return pp::Token::OP_AND; 169 } 170 "^^" { 171 yylval->assign(yytext, yyleng); 172 return pp::Token::OP_XOR; 173 } 174 "||" { 175 yylval->assign(yytext, yyleng); 176 return pp::Token::OP_OR; 177 } 178 "+=" { 179 yylval->assign(yytext, yyleng); 180 return pp::Token::OP_ADD_ASSIGN; 181 } 182 "-=" { 183 yylval->assign(yytext, yyleng); 184 return pp::Token::OP_SUB_ASSIGN; 185 } 186 "*=" { 187 yylval->assign(yytext, yyleng); 188 return pp::Token::OP_MUL_ASSIGN; 189 } 190 "/=" { 191 yylval->assign(yytext, yyleng); 192 return pp::Token::OP_DIV_ASSIGN; 193 } 194 "%=" { 195 yylval->assign(yytext, yyleng); 196 return pp::Token::OP_MOD_ASSIGN; 197 } 198 "<<=" { 199 yylval->assign(yytext, yyleng); 200 return pp::Token::OP_LEFT_ASSIGN; 201 } 202 ">>=" { 203 yylval->assign(yytext, yyleng); 204 return pp::Token::OP_RIGHT_ASSIGN; 205 } 206 "&=" { 207 yylval->assign(yytext, yyleng); 208 return pp::Token::OP_AND_ASSIGN; 209 } 210 "^=" { 211 yylval->assign(yytext, yyleng); 212 return pp::Token::OP_XOR_ASSIGN; 213 } 214 "|=" { 215 yylval->assign(yytext, yyleng); 216 return pp::Token::OP_OR_ASSIGN; 217 } 218 219 {PUNCTUATOR} { 220 yylval->assign(1, yytext[0]); 221 return yytext[0]; 222 } 223 224 [ \t\v\f]+ { yyextra->leadingSpace = true; } 225 226 {NEWLINE} { 227 ++yylineno; 228 yylval->assign(1, '\n'); 229 return '\n'; 230 } 231 232 \\{NEWLINE} { ++yylineno; } 233 234 . { 235 yylval->assign(1, yytext[0]); 236 return pp::Token::PP_OTHER; 237 } 238 239 <*><<EOF>> { 240 // YY_USER_ACTION is not invoked for handling EOF. 241 // Set the location for EOF token manually. 242 pp::Input* input = &yyextra->input; 243 pp::Input::Location* scanLoc = &yyextra->scanLoc; 244 yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0; 245 if (scanLoc->sIndex != sIndexMax) 246 { 247 // We can only reach here if there are empty strings at the 248 // end of the input. 249 scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0; 250 // FIXME: this is not 64-bit clean. 251 yyfileno = static_cast<int>(sIndexMax); yylineno = 1; 252 } 253 yylloc->file = yyfileno; 254 yylloc->line = yylineno; 255 yylval->clear(); 256 257 if (YY_START == COMMENT) 258 { 259 yyextra->diagnostics->report(pp::Diagnostics::PP_EOF_IN_COMMENT, 260 pp::SourceLocation(yyfileno, yylineno), 261 ""); 262 } 263 yyterminate(); 264 } 265 266 %% 267 268 namespace pp { 269 270 Tokenizer::Tokenizer(Diagnostics *diagnostics) : mHandle(0) 271 { 272 mContext.diagnostics = diagnostics; 273 } 274 275 Tokenizer::~Tokenizer() 276 { 277 destroyScanner(); 278 } 279 280 bool Tokenizer::init(size_t count, const char * const string[], const int length[]) 281 { 282 if ((count > 0) && (string == 0)) 283 return false; 284 285 mContext.input = Input(count, string, length); 286 return initScanner(); 287 } 288 289 void Tokenizer::setFileNumber(int file) 290 { 291 // We use column number as file number. 292 // See macro yyfileno. 293 yyset_column(file, mHandle); 294 } 295 296 void Tokenizer::setLineNumber(int line) 297 { 298 yyset_lineno(line, mHandle); 299 } 300 301 void Tokenizer::setMaxTokenSize(size_t maxTokenSize) 302 { 303 mMaxTokenSize = maxTokenSize; 304 } 305 306 void Tokenizer::lex(Token *token) 307 { 308 token->type = yylex(&token->text, &token->location, mHandle); 309 if (token->text.size() > mMaxTokenSize) 310 { 311 mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG, 312 token->location, token->text); 313 token->text.erase(mMaxTokenSize); 314 } 315 316 token->flags = 0; 317 318 token->setAtStartOfLine(mContext.lineStart); 319 mContext.lineStart = token->type == '\n'; 320 321 token->setHasLeadingSpace(mContext.leadingSpace); 322 mContext.leadingSpace = false; 323 } 324 325 bool Tokenizer::initScanner() 326 { 327 if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle)) 328 return false; 329 330 yyrestart(0, mHandle); 331 return true; 332 } 333 334 void Tokenizer::destroyScanner() 335 { 336 if (mHandle == NULL) 337 return; 338 339 yylex_destroy(mHandle); 340 mHandle = NULL; 341 } 342 343 } // namespace pp 344 345