OpenGrok
Home
Sort by relevance
Sort by last modified time
Full Search
Definition
Symbol
File Path
History
|
|
Help
Searched
full:token_span
(Results
1 - 6
of
6
) sorted by null
/external/libtextclassifier/
feature-processor.cc
215
TokenSpan
token_span
;
local
216
if (!LabelToTokenSpan(label, &
token_span
)) {
220
const int result_begin_token_index =
token_span
.first;
224
const int result_end_token_index =
token_span
.second;
259
TokenSpan*
token_span
) const {
261
*
token_span
= label_to_selection_[label];
370
const std::vector<Token>& selectable_tokens, TokenSpan
token_span
) {
371
return {selectable_tokens[
token_span
.first].start,
372
selectable_tokens[
token_span
.second - 1].end};
617
const TokenSpan&
token_span
, const std::vector<Token>& tokens) const
[
all
...]
feature-processor_test.cc
320
TokenSpan
token_span
;
local
321
feature_processor.LabelToTokenSpan(label, &
token_span
);
322
EXPECT_EQ(0,
token_span
.first);
323
EXPECT_EQ(0,
token_span
.second);
357
feature_processor3.LabelToTokenSpan(label2, &
token_span
);
358
EXPECT_EQ(1,
token_span
.first);
359
EXPECT_EQ(0,
token_span
.second);
395
TokenSpan
token_span
;
local
396
feature_processor.LabelToTokenSpan(label, &
token_span
);
397
EXPECT_EQ(0,
token_span
.first)
[
all
...]
types.h
94
inline int TokenSpanSize(const TokenSpan&
token_span
) {
95
return
token_span
.second -
token_span
.first;
113
inline TokenSpan ExpandTokenSpan(const TokenSpan&
token_span
,
115
return {
token_span
.first - num_tokens_left,
116
token_span
.second + num_tokens_right};
feature-processor.h
84
const std::vector<Token>& selectable_tokens, TokenSpan
token_span
);
133
bool LabelToTokenSpan(int label, TokenSpan*
token_span
) const;
165
TokenSpan
token_span
) const;
169
bool ExtractFeatures(const std::vector<Token>& tokens, TokenSpan
token_span
,
246
float SupportedCodepointsRatio(const TokenSpan&
token_span
,
text-classifier.cc
[
all
...]
text-classifier.h
166
TokenSpan
token_span
;
member in struct:libtextclassifier2::TextClassifier::ScoredChunk
Completed in 198 milliseconds