Lines Matching refs:Tokenizer
53 class Tokenizer;
91 class LIBPROTOBUF_EXPORT Tokenizer {
93 // Construct a Tokenizer that reads and tokenizes text from the given
96 Tokenizer(ZeroCopyInputStream* input, ErrorCollector* error_collector);
97 ~Tokenizer();
200 // comes from a TYPE_FLOAT token parsed by Tokenizer. If it doesn't, the
205 // comes from a TYPE_STRING token parsed by Tokenizer. If it doesn't, the
215 // parsed by a Tokenizer, the result is undefined (possibly an assert
257 GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Tokenizer);
394 inline const Tokenizer::Token& Tokenizer::current() {
398 inline const Tokenizer::Token& Tokenizer::previous() {
402 inline void Tokenizer
