1 // Copyright 2016 The SwiftShader Authors. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 #ifndef COMPILER_PREPROCESSOR_TOKENIZER_H_ 16 #define COMPILER_PREPROCESSOR_TOKENIZER_H_ 17 18 #include "Input.h" 19 #include "Lexer.h" 20 #include "pp_utils.h" 21 22 #include <algorithm> 23 24 namespace pp 25 { 26 27 class Diagnostics; 28 29 class Tokenizer : public Lexer 30 { 31 public: 32 struct Context 33 { 34 Diagnostics* diagnostics; 35 36 Input input; 37 // The location where yytext points to. Token location should track 38 // scanLoc instead of Input::mReadLoc because they may not be the same 39 // if text is buffered up in the scanner input buffer. 40 Input::Location scanLoc; 41 42 bool leadingSpace; 43 bool lineStart; 44 }; 45 static const size_t kMaxTokenLength; 46 47 Tokenizer(Diagnostics* diagnostics); 48 ~Tokenizer(); 49 50 bool init(int count, const char* const string[], const int length[]); 51 52 void setFileNumber(int file); 53 void setLineNumber(int line); 54 55 virtual void lex(Token* token); 56 57 private: 58 PP_DISALLOW_COPY_AND_ASSIGN(Tokenizer); 59 bool initScanner(); 60 void destroyScanner(); 61 62 void* mHandle; // Scanner handle. 63 Context mContext; // Scanner extra. 64 }; 65 66 } // namespace pp 67 #endif // COMPILER_PREPROCESSOR_TOKENIZER_H_ 68 69
