Lines Matching refs:token
38 // Whether to extract the token case feature.
64 // Extracts features from a token.
65 // - is_in_span is a bool indicator whether the token is a part of the
70 // (depending on the options) for the token
71 bool Extract(const Token& token, bool is_in_span,
81 // Hashes given token to given number of buckets.
82 int HashToken(StringPiece token) const;
84 // Extracts the charactergram features from the token.
85 std::vector<int> ExtractCharactergramFeatures(const Token& token) const;
87 // Extracts the charactergram features from the token in a non-unicode-aware
89 std::vector<int> ExtractCharactergramFeaturesAscii(const Token& token) const;
91 // Extracts the charactergram features from the token in a unicode-aware way.
93 const Token& token) const;
