1 // 2 // Copyright (c) 2012 The ANGLE Project Authors. All rights reserved. 3 // Use of this source code is governed by a BSD-style license that can be 4 // found in the LICENSE file. 5 // 6 7 #ifndef COMPILER_PREPROCESSOR_TOKENIZER_H_ 8 #define COMPILER_PREPROCESSOR_TOKENIZER_H_ 9 10 #include "Input.h" 11 #include "Lexer.h" 12 #include "pp_utils.h" 13 14 namespace pp 15 { 16 17 class Diagnostics; 18 19 class Tokenizer : public Lexer 20 { 21 public: 22 struct Context 23 { 24 Diagnostics* diagnostics; 25 26 Input input; 27 // The location where yytext points to. Token location should track 28 // scanLoc instead of Input::mReadLoc because they may not be the same 29 // if text is buffered up in the scanner input buffer. 30 Input::Location scanLoc; 31 32 bool leadingSpace; 33 bool lineStart; 34 }; 35 static const std::size_t kMaxTokenLength; 36 37 Tokenizer(Diagnostics* diagnostics); 38 ~Tokenizer(); 39 40 bool init(size_t count, const char* const string[], const int length[]); 41 42 void setFileNumber(int file); 43 void setLineNumber(int line); 44 45 virtual void lex(Token* token); 46 47 private: 48 PP_DISALLOW_COPY_AND_ASSIGN(Tokenizer); 49 bool initScanner(); 50 void destroyScanner(); 51 52 void* mHandle; // Scanner handle. 53 Context mContext; // Scanner extra. 54 }; 55 56 } // namespace pp 57 #endif // COMPILER_PREPROCESSOR_TOKENIZER_H_ 58 59
