Lines Matching refs:extractor
17 #include "smartselect/token-feature-extractor.h"
38 TestingTokenFeatureExtractor extractor(options);
43 extractor.Extract(Token{"Hello", 0, 5}, true, &sparse_features,
49 extractor.HashToken("H"),
50 extractor.HashToken("e"),
51 extractor.HashToken("l"),
52 extractor.HashToken("l"),
53 extractor.HashToken("o"),
54 extractor.HashToken("^H"),
55 extractor.HashToken("He"),
56 extractor.HashToken("el"),
57 extractor.HashToken("ll"),
58 extractor.HashToken("lo"),
59 extractor.HashToken("o$"),
60 extractor.HashToken("^He"),
61 extractor.HashToken("Hel"),
62 extractor.HashToken("ell"),
63 extractor.HashToken("llo"),
64 extractor.HashToken("lo$")
71 extractor.Extract(Token{"world!", 23, 29}, false, &sparse_features,
77 extractor.HashToken("w"),
78 extractor.HashToken("o"),
79 extractor.HashToken("r"),
80 extractor.HashToken("l"),
81 extractor.HashToken("d"),
82 extractor.HashToken("!"),
83 extractor.HashToken("^w"),
84 extractor.HashToken("wo"),
85 extractor.HashToken("or"),
86 extractor.HashToken("rl"),
87 extractor.HashToken("ld"),
88 extractor.HashToken("d!"),
89 extractor.HashToken("!$"),
90 extractor.HashToken("^wo"),
91 extractor.HashToken("wor"),
92 extractor.HashToken("orl"),
93 extractor.HashToken("rld"),
94 extractor.HashToken("ld!"),
95 extractor.HashToken("d!$"),
108 TestingTokenFeatureExtractor extractor(options);
113 extractor.Extract(Token{"H?lló", 0, 5}, true, &sparse_features,
119 extractor.HashToken("H"),
120 extractor.HashToken("?"),
121 extractor.HashToken("l"),
122 extractor.HashToken("l"),
123 extractor.HashToken("ó"),
124 extractor.HashToken("^H"),
125 extractor.HashToken("H?"),
126 extractor.HashToken("?l"),
127 extractor.HashToken("ll"),
128 extractor.HashToken("ló"),
129 extractor.HashToken("ó$"),
130 extractor.HashToken("^H?"),
131 extractor.HashToken("H?l"),
132 extractor.HashToken("?ll"),
133 extractor.HashToken("lló"),
134 extractor.HashToken("ló$")
141 extractor.Extract(Token{"world!", 23, 29}, false, &sparse_features,
147 extractor.HashToken("w"),
148 extractor.HashToken("o"),
149 extractor.HashToken("r"),
150 extractor.HashToken("l"),
151 extractor.HashToken("d"),
152 extractor.HashToken("!"),
153 extractor.HashToken("^w"),
154 extractor.HashToken("wo"),
155 extractor.HashToken("or"),
156 extractor.HashToken("rl"),
157 extractor.HashToken("ld"),
158 extractor.HashToken("d!"),
159 extractor.HashToken("!$"),
160 extractor.HashToken("^wo"),
161 extractor.HashToken("wor"),
162 extractor.HashToken("orl"),
163 extractor.HashToken("rld"),
164 extractor.HashToken("ld!"),
165 extractor.HashToken("d!$"),
178 TokenFeatureExtractor extractor(options);
182 extractor.Extract(Token{"H?lló", 0, 5}, true, &sparse_features,
188 extractor.Extract(Token{"world!", 23, 29}, false, &sparse_features,
194 extractor.Extract(Token{"?", 23, 29}, false, &sparse_features,
200 extractor.Extract(Token{"?", 23, 29}, false, &sparse_features,
211 TokenFeatureExtractor extractor(options);
215 extractor.Extract(Token{"9:30am", 0, 6}, true, &sparse_features,
219 extractor.Extract(Token{"5:32am", 0, 6}, true, &sparse_features2,
223 extractor.Extract(Token{"10:32am", 0, 6}, true, &sparse_features2,
235 TokenFeatureExtractor extractor(options);
239 extractor.Extract(Token{"9:30am", 0, 6}, true, &sparse_features,
243 extractor.Extract(Token{"5:32am", 0, 6}, true, &sparse_features2,
247 extractor.Extract(Token{"10:32am", 0, 6}, true, &sparse_features2,
259 TokenFeatureExtractor extractor(options);
263 extractor.Extract(Token{"AABB", 0, 6}, true, &sparse_features,
267 extractor.Extract(Token{"aaBB", 0, 6}, true, &sparse_features2,
271 extractor.Extract(Token{"aAbB", 0, 6}, true, &sparse_features2,
282 TokenFeatureExtractor extractor(options);
286 extractor.Extract(Token{"??", 0, 6}, true, &sparse_features, &dense_features);
289 extractor.Extract(Token{"??", 0, 6}, true, &sparse_features2,
302 TokenFeatureExtractor extractor(options);
306 extractor.Extract(Token{"abCde", 0, 6}, true, &sparse_features,
311 extractor.Extract(Token{"abcde", 0, 6}, true, &sparse_features,
316 extractor.Extract(Token{"12c45", 0, 6}, true, &sparse_features,
321 extractor.Extract(Token{"12345", 0, 6}, true, &sparse_features,
333 TestingTokenFeatureExtractor extractor(options);
338 extractor.Extract(Token{"abcdefghijklmnopq?stuvwxyz", 0, 0}, true,
344 extractor.HashToken("^abcdefghij\1q?stuvwxyz"),
345 extractor.HashToken("abcdefghij\1q?stuvwxyz$"),
391 TestingTokenFeatureExtractor extractor(options);
396 extractor.Extract(Token(), false, &sparse_features, &dense_features);
399 testing::ElementsAreArray({extractor.HashToken("<PAD>")}));
