Lines Matching refs:Tokenizer
17 #include "smartselect/tokenizer.h"
27 class TestingTokenizer : public Tokenizer {
31 : Tokenizer(codepoint_range_configs) {}
60 TestingTokenizer tokenizer(configs);
63 EXPECT_EQ(tokenizer.TestFindTokenizationRole(0),
65 EXPECT_EQ(tokenizer.TestFindTokenizationRole(5),
67 EXPECT_EQ(tokenizer.TestFindTokenizationRole(10),
71 EXPECT_EQ(tokenizer.TestFindTokenizationRole(31),
73 EXPECT_EQ(tokenizer.TestFindTokenizationRole(32),
75 EXPECT_EQ(tokenizer.TestFindTokenizationRole(33),
79 EXPECT_EQ(tokenizer.TestFindTokenizationRole(1233),
81 EXPECT_EQ(tokenizer.TestFindTokenizationRole(1234),
83 EXPECT_EQ(tokenizer.TestFindTokenizationRole(12344),
85 EXPECT_EQ(tokenizer.TestFindTokenizationRole(12345),
89 EXPECT_EQ(tokenizer.TestFindTokenizationRole(99),
104 TestingTokenizer tokenizer(configs);
105 std::vector<Token> tokens = tokenizer.Tokenize("Hello world!");
235 Tokenizer tokenizer(configs);
238 tokens = tokenizer.Tokenize(
242 tokens = tokenizer.Tokenize("??? hello ???????");
