Lines Matching full:tokenizer
8 #include "tools/gn/tokenizer.h"
23 std::vector<Token> results = Tokenizer::Tokenize(&input_file, &err);
38 TEST(Tokenizer, Empty) {
43 std::vector<Token> results = Tokenizer::Tokenize(&empty_string_input, &err);
49 results = Tokenizer::Tokenize(&whitespace_input, &err);
53 TEST(Tokenizer, Identifier) {
60 TEST(Tokenizer, Integer) {
68 TEST(Tokenizer, String) {
78 TEST(Tokenizer, Operator) {
96 TEST(Tokenizer, Scoper) {
108 TEST(Tokenizer, FunctionCall) {
123 TEST(Tokenizer, StringUnescaping) {
127 std::vector<Token> results = Tokenizer::Tokenize(&input, &err);
134 TEST(Tokenizer, Locations) {
138 std::vector<Token> results = Tokenizer::Tokenize(&input, &err);
147 TEST(Tokenizer, ByteOffsetOfNthLine) {
148 EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine("foo", 1));
153 EXPECT_EQ('x', input1[Tokenizer::ByteOffsetOfNthLine(input1, 2)]);
154 EXPECT_EQ('y', input1[Tokenizer::ByteOffsetOfNthLine(input1, 4)]);
160 EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine(input2, 1));
161 EXPECT_EQ(2u, Tokenizer::ByteOffsetOfNthLine(input2, 2));
