HomeSort by relevance Sort by last modified time
    Searched refs:text_unicode (Results 1 - 4 of 4) sorted by null

  /external/libtextclassifier/
tokenizer.cc 80 UnicodeText text_unicode = UTF8ToUnicodeText(text, /*do_copy=*/false); local
81 return Tokenize(text_unicode);
84 std::vector<Token> Tokenizer::Tokenize(const UnicodeText& text_unicode) const {
90 for (auto it = text_unicode.begin(); it != text_unicode.end();
tokenizer.h 45 std::vector<Token> Tokenize(const UnicodeText& text_unicode) const;
feature-processor.cc 179 const UnicodeText text_unicode = UTF8ToUnicodeText(text, /*do_copy=*/false); local
180 return Tokenize(text_unicode);
184 const UnicodeText& text_unicode) const {
187 return tokenizer_.Tokenize(text_unicode);
193 if (!ICUTokenize(text_unicode, &result)) {
198 InternalRetokenize(text_unicode, &result);
204 return tokenizer_.Tokenize(text_unicode);
    [all...]
feature-processor.h 130 std::vector<Token> Tokenize(const UnicodeText& text_unicode) const;

Completed in 3493 milliseconds