OpenGrok
Home
Sort by relevance
Sort by last modified time
Full Search
Definition
Symbol
File Path
History
|
|
Help
Searched
refs:token_span
(Results
1 - 6
of
6
) sorted by null
/external/libtextclassifier/annotator/
feature-processor.cc
207
TokenSpan
token_span
;
local
208
if (!LabelToTokenSpan(label, &
token_span
)) {
212
const int result_begin_token_index =
token_span
.first;
216
const int result_end_token_index =
token_span
.second;
251
TokenSpan*
token_span
) const {
253
*
token_span
= label_to_selection_[label];
362
const std::vector<Token>& selectable_tokens, TokenSpan
token_span
) {
363
return {selectable_tokens[
token_span
.first].start,
364
selectable_tokens[
token_span
.second - 1].end};
600
const TokenSpan&
token_span
, const std::vector<Token>& tokens) const
[
all
...]
feature-processor.h
82
const std::vector<Token>& selectable_tokens, TokenSpan
token_span
);
115
bool LabelToTokenSpan(int label, TokenSpan*
token_span
) const;
147
TokenSpan
token_span
) const;
151
bool ExtractFeatures(const std::vector<Token>& tokens, TokenSpan
token_span
,
225
float SupportedCodepointsRatio(const TokenSpan&
token_span
,
types.h
99
inline int TokenSpanSize(const TokenSpan&
token_span
) {
100
return
token_span
.second -
token_span
.first;
118
inline TokenSpan ExpandTokenSpan(const TokenSpan&
token_span
,
120
return {
token_span
.first - num_tokens_left,
121
token_span
.second + num_tokens_right};
feature-processor_test.cc
318
TokenSpan
token_span
;
local
319
feature_processor.LabelToTokenSpan(label, &
token_span
);
320
EXPECT_EQ(0,
token_span
.first);
321
EXPECT_EQ(0,
token_span
.second);
355
feature_processor3.LabelToTokenSpan(label2, &
token_span
);
356
EXPECT_EQ(1,
token_span
.first);
357
EXPECT_EQ(0,
token_span
.second);
392
TokenSpan
token_span
;
local
393
feature_processor.LabelToTokenSpan(label, &
token_span
);
394
EXPECT_EQ(0,
token_span
.first)
[
all
...]
annotator.h
276
TokenSpan
token_span
;
member in struct:libtextclassifier3::Annotator::ScoredChunk
annotator.cc
[
all
...]
Completed in 9007 milliseconds