Lines Matching refs:token_span
207 TokenSpan token_span;
208 if (!LabelToTokenSpan(label, &token_span)) {
212 const int result_begin_token_index = token_span.first;
216 const int result_end_token_index = token_span.second;
251 TokenSpan* token_span) const {
253 *token_span = label_to_selection_[label];
362 const std::vector<Token>& selectable_tokens, TokenSpan token_span) {
363 return {selectable_tokens[token_span.first].start,
364 selectable_tokens[token_span.second - 1].end};
600 const TokenSpan& token_span, const std::vector<Token>& tokens) const {
603 for (int i = token_span.first; i < token_span.second; ++i) {
662 TokenSpan token_span{l, r};
663 selection_to_label_[token_span] = selection_label_id;
664 label_to_selection_.push_back(token_span);
747 const std::vector<Token>& tokens, TokenSpan token_span) const {
750 SupportedCodepointsRatio(token_span, tokens);
761 const std::vector<Token>& tokens, TokenSpan token_span,
767 features->reserve(feature_vector_size * TokenSpanSize(token_span));
768 for (int i = token_span.first; i < token_span.second; ++i) {
787 *cached_features = CachedFeatures::Create(token_span, std::move(features),
