HomeSort by relevance Sort by last modified time
    Searched refs:tokenizer_ (Results 1 - 3 of 3) sorted by null

  /external/protobuf/src/google/protobuf/
text_format.cc 242 tokenizer_(input_stream, &tokenizer_error_collector_),
252 tokenizer_.set_allow_f_after_float(true);
255 tokenizer_.set_comment_style(io::Tokenizer::SH_COMMENT_STYLE);
258 tokenizer_.set_require_space_after_number(false);
259 tokenizer_.set_allow_multiline_strings(true);
263 tokenizer_.Next();
333 ReportError(tokenizer_.current().line, tokenizer_.current().column,
340 ReportWarning(tokenizer_.current().line, tokenizer_.current().column
1094 io::Tokenizer tokenizer_; member in class:google::protobuf::TextFormat::Parser::ParserImpl
    [all...]
  /external/libtextclassifier/smartselect/
feature-processor.h 97 tokenizer_({options.tokenization_codepoint_config().begin(),
252 Tokenizer tokenizer_; member in class:libtextclassifier::FeatureProcessor
feature-processor.cc 187 return tokenizer_.Tokenize(utf8_text);
204 return tokenizer_.Tokenize(utf8_text);
760 std::vector<Token> tokens = tokenizer_.Tokenize(text);

Completed in 356 milliseconds