1 // Copyright 2016 The SwiftShader Authors. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 #ifndef COMPILER_PREPROCESSOR_TOKENIZER_H_ 16 #define COMPILER_PREPROCESSOR_TOKENIZER_H_ 17 18 #include "Input.h" 19 #include "Lexer.h" 20 #include "pp_utils.h" 21 22 namespace pp 23 { 24 25 class Diagnostics; 26 27 class Tokenizer : public Lexer 28 { 29 public: 30 struct Context 31 { 32 Diagnostics *diagnostics; 33 34 Input input; 35 // The location where yytext points to. Token location should track 36 // scanLoc instead of Input::mReadLoc because they may not be the same 37 // if text is buffered up in the scanner input buffer. 38 Input::Location scanLoc; 39 40 bool leadingSpace; 41 bool lineStart; 42 }; 43 44 Tokenizer(Diagnostics *diagnostics); 45 ~Tokenizer() override; 46 47 bool init(size_t count, const char *const string[], const int length[]); 48 49 void setFileNumber(int file); 50 void setLineNumber(int line); 51 void setMaxTokenSize(size_t maxTokenSize); 52 53 void lex(Token *token) override; 54 55 private: 56 PP_DISALLOW_COPY_AND_ASSIGN(Tokenizer); 57 bool initScanner(); 58 void destroyScanner(); 59 60 void *mHandle; // Scanner handle. 61 Context mContext; // Scanner extra. 62 size_t mMaxTokenSize; // Maximum token size 63 }; 64 65 } // namespace pp 66 #endif // COMPILER_PREPROCESSOR_TOKENIZER_H_ 67 68
