Lines Matching refs:Tokenizer
40 #include <google/protobuf/io/tokenizer.h>
184 EXPECT_TRUE(Tokenizer::ParseInteger(text, kuint64max, &result));
199 Tokenizer::TokenType type;
209 { "hello", Tokenizer::TYPE_IDENTIFIER },
212 { "123", Tokenizer::TYPE_INTEGER },
213 { "0xab6", Tokenizer::TYPE_INTEGER },
214 { "0XAB6", Tokenizer::TYPE_INTEGER },
215 { "0X1234567", Tokenizer::TYPE_INTEGER },
216 { "0x89abcdef", Tokenizer::TYPE_INTEGER },
217 { "0x89ABCDEF", Tokenizer::TYPE_INTEGER },
218 { "01234567", Tokenizer::TYPE_INTEGER },
221 { "123.45", Tokenizer::TYPE_FLOAT },
222 { "1.", Tokenizer::TYPE_FLOAT },
223 { "1e3", Tokenizer::TYPE_FLOAT },
224 { "1E3", Tokenizer::TYPE_FLOAT },
225 { "1e-3", Tokenizer::TYPE_FLOAT },
226 { "1e+3", Tokenizer::TYPE_FLOAT },
227 { "1.e3", Tokenizer::TYPE_FLOAT },
228 { "1.2e3", Tokenizer::TYPE_FLOAT },
229 { ".1", Tokenizer::TYPE_FLOAT },
230 { ".1e3", Tokenizer::TYPE_FLOAT },
231 { ".1e-3", Tokenizer::TYPE_FLOAT },
232 { ".1e+3", Tokenizer::TYPE_FLOAT },
235 { "'hello'", Tokenizer::TYPE_STRING },
236 { "\"foo\"", Tokenizer::TYPE_STRING },
237 { "'a\"b'", Tokenizer::TYPE_STRING },
238 { "\"a'b\"", Tokenizer::TYPE_STRING },
239 { "'a\\'b'", Tokenizer::TYPE_STRING },
240 { "\"a\\\"b\"", Tokenizer::TYPE_STRING },
241 { "'\\xf'", Tokenizer::TYPE_STRING },
242 { "'\\0'", Tokenizer::TYPE_STRING },
245 { "+", Tokenizer::TYPE_SYMBOL },
246 { ".", Tokenizer::TYPE_SYMBOL },
250 // Set up the tokenizer.
255 Tokenizer tokenizer(&input, &error_collector);
258 EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type);
259 EXPECT_EQ("", tokenizer.current().text);
260 EXPECT_EQ(0, tokenizer.current().line);
261 EXPECT_EQ(0, tokenizer.current().column);
262 EXPECT_EQ(0, tokenizer.current().end_column);
265 ASSERT_TRUE(tokenizer.Next());
268 EXPECT_EQ(kSimpleTokenCases_case.type, tokenizer.current().type);
270 EXPECT_EQ(kSimpleTokenCases_case.input, tokenizer.current().text);
272 EXPECT_EQ(0, tokenizer.current().line);
273 EXPECT_EQ(0, tokenizer.current().column);
275 tokenizer.current().end_column);
278 EXPECT_FALSE(tokenizer.Next());
281 EXPECT_EQ(Tokenizer::TYPE_END, tokenizer.current().type);
282 EXPECT_EQ("", tokenizer.current().text);
283 EXPECT_EQ(0, tokenizer.current().line);
284 EXPECT_EQ(kSimpleTokenCases_case.input.size(), tokenizer.current().column);
286 tokenizer.current().end_column);
295 // Set up the tokenizer.
299 Tokenizer tokenizer(&input, &error_collector);
300 tokenizer.set_allow_f_after_float(true);
303 ASSERT_TRUE(tokenizer.Next());
304 EXPECT_EQ(tokenizer.current().text, "1f");
305 EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
306 ASSERT_TRUE(tokenizer.Next());
307 EXPECT_EQ(tokenizer.current().text, "2.5f");
308 EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
309 ASSERT_TRUE(tokenizer.Next());
310 EXPECT_EQ(tokenizer.current().text, "6e3f");
311 EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
312 ASSERT_TRUE(tokenizer.Next());
313 EXPECT_EQ(tokenizer.current().text, "7F");
314 EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
317 EXPECT_FALSE(tokenizer.Next());
330 Tokenizer::Token output[10]; // The compiler wants a constant array
344 { Tokenizer::TYPE_END , "" , 0, 0 },
349 { Tokenizer::TYPE_IDENTIFIER, "foo" , 0, 0, 3 },
350 { Tokenizer::TYPE_INTEGER , "1" , 0, 4, 5 },
351 { Tokenizer::TYPE_FLOAT , "1.2" , 0, 6, 9 },
352 { Tokenizer::TYPE_SYMBOL , "+" , 0, 10, 11 },
353 { Tokenizer::TYPE_STRING , "'bar'", 0, 12, 17 },
354 { Tokenizer::TYPE_END , "" , 0, 17, 17 },
359 { Tokenizer::TYPE_SYMBOL , "!" , 0, 0, 1 },
360 { Tokenizer::TYPE_SYMBOL , "@" , 0, 1, 2 },
361 { Tokenizer::TYPE_SYMBOL , "+" , 0, 2, 3 },
362 { Tokenizer::TYPE_SYMBOL , "%" , 0, 3, 4 },
363 { Tokenizer::TYPE_END , "" , 0, 4, 4 },
368 { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3 },
369 { Tokenizer::TYPE_IDENTIFIER, "bar", 0, 4, 7 },
370 { Tokenizer::TYPE_IDENTIFIER, "rab", 1, 0, 3 },
371 { Tokenizer::TYPE_IDENTIFIER, "oof", 1, 4, 7 },
372 { Tokenizer::TYPE_END , "" , 1, 7, 7 },
377 { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3 },
378 { Tokenizer::TYPE_IDENTIFIER, "bar", 0, 8, 11 },
379 { Tokenizer::TYPE_IDENTIFIER, "baz", 0, 16, 19 },
380 { Tokenizer::TYPE_END , "" , 0, 19, 19 },
385 { Tokenizer::TYPE_STRING , "\"foo\tbar\"", 0, 0, 12 },
386 { Tokenizer::TYPE_IDENTIFIER, "baz" , 0, 13, 16 },
387 { Tokenizer::TYPE_END , "" , 0, 16, 16 },
393 { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3 },
394 { Tokenizer::TYPE_IDENTIFIER, "bar", 1, 0, 3 },
395 { Tokenizer
400 { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3 },
401 { Tokenizer::TYPE_IDENTIFIER, "bar", 0, 34, 37 },
402 { Tokenizer::TYPE_END , "" , 0, 37, 37 },
408 { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3 },
409 { Tokenizer::TYPE_SYMBOL , "#" , 0, 4, 5 },
410 { Tokenizer::TYPE_IDENTIFIER, "bar", 0, 6, 9 },
411 { Tokenizer::TYPE_IDENTIFIER, "baz", 1, 0, 3 },
412 { Tokenizer::TYPE_END , "" , 1, 3, 3 },
417 { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3 },
418 { Tokenizer::TYPE_IDENTIFIER, "bar", 1, 11, 14 },
419 { Tokenizer::TYPE_END , "" , 1, 14, 14 },
424 // Set up the tokenizer.
429 Tokenizer tokenizer(&input, &error_collector);
432 EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type);
433 EXPECT_EQ("", tokenizer.current().text);
434 EXPECT_EQ(0, tokenizer.current().line);
435 EXPECT_EQ(0, tokenizer.current().column);
436 EXPECT_EQ(0, tokenizer.current().end_column);
440 Tokenizer::Token token;
446 Tokenizer::Token previous = tokenizer.current();
449 if (token.type != Tokenizer::TYPE_END) {
450 ASSERT_TRUE(tokenizer.Next());
452 ASSERT_FALSE(tokenizer.Next());
456 EXPECT_EQ(previous.type, tokenizer.previous().type);
457 EXPECT_EQ(previous.text, tokenizer.previous().text);
458 EXPECT_EQ(previous.line, tokenizer.previous().line);
459 EXPECT_EQ(previous.column, tokenizer.previous().column);
460 EXPECT_EQ(previous.end_column, tokenizer.previous().end_column);
463 EXPECT_EQ(token.type, tokenizer.current().type);
464 EXPECT_EQ(token.text, tokenizer.current().text);
465 EXPECT_EQ(token.line, tokenizer.current().line);
466 EXPECT_EQ(token.column, tokenizer.current().column);
467 EXPECT_EQ(token.end_column, tokenizer.current().end_column);
469 } while (token.type != Tokenizer::TYPE_END);
491 // Set up the tokenizer.
494 Tokenizer tokenizer(&input, &error_collector);
495 tokenizer.set_comment_style(Tokenizer::SH_COMMENT_STYLE);
499 EXPECT_TRUE(tokenizer.Next());
500 EXPECT_EQ(tokenizer.current().text, kTokens[i]);
504 EXPECT_FALSE(tokenizer.Next());
676 // Set up the tokenizer.
681 Tokenizer tokenizer(&input, &error_collector);
683 // Set up a second tokenizer where we'll pass all NULLs to NextWithComments().
687 Tokenizer tokenizer2(&input2, &error_collector);
689 tokenizer.Next();
692 EXPECT_EQ("prev", tokenizer.current().text);
698 tokenizer.NextWithComments(&prev_trailing_comments, &detached_comments,
701 EXPECT_EQ("next", tokenizer.current().text);
741 EXPECT_FALSE(Tokenizer::ParseInteger("zxy", kuint64max, &i));
742 EXPECT_FALSE(Tokenizer::ParseInteger("1.2", kuint64max, &i));
743 EXPECT_FALSE(Tokenizer::ParseInteger("08", kuint64max, &i));
744 EXPECT_FALSE(Tokenizer::ParseInteger("0xg", kuint64max, &i));
745 EXPECT_FALSE(Tokenizer::ParseInteger("-1", kuint64max, &i));
748 EXPECT_TRUE (Tokenizer::ParseInteger("0", 0, &i));
749 EXPECT_FALSE(Tokenizer::ParseInteger("1", 0, &i));
750 EXPECT_TRUE (Tokenizer::ParseInteger("1", 1, &i));
751 EXPECT_TRUE (Tokenizer::ParseInteger("12345", 12345, &i));
752 EXPECT_FALSE(Tokenizer::ParseInteger("12346", 12345, &i));
753 EXPECT_TRUE (Tokenizer::ParseInteger("0xFFFFFFFFFFFFFFFF" , kuint64max, &i));
754 EXPECT_FALSE(Tokenizer::ParseInteger("0x10000000000000000", kuint64max, &i));
758 EXPECT_DOUBLE_EQ(1 , Tokenizer::ParseFloat("1."));
759 EXPECT_DOUBLE_EQ(1e3 , Tokenizer::ParseFloat("1e3"));
760 EXPECT_DOUBLE_EQ(1e3 , Tokenizer::ParseFloat("1E3"));
761 EXPECT_DOUBLE_EQ(1.5e3, Tokenizer::ParseFloat("1.5e3"));
762 EXPECT_DOUBLE_EQ(.1 , Tokenizer::ParseFloat(".1"));
763 EXPECT_DOUBLE_EQ(.25 , Tokenizer::ParseFloat(".25"));
764 EXPECT_DOUBLE_EQ(.1e3 , Tokenizer::ParseFloat(".1e3"));
765 EXPECT_DOUBLE_EQ(.25e3, Tokenizer::ParseFloat(".25e3"));
766 EXPECT_DOUBLE_EQ(.1e+3, Tokenizer::ParseFloat(".1e+3"));
767 EXPECT_DOUBLE_EQ(.1e-3, Tokenizer::ParseFloat(".1e-3"));
768 EXPECT_DOUBLE_EQ(5 , Tokenizer::ParseFloat("5"));
769 EXPECT_DOUBLE_EQ(6e-12, Tokenizer::ParseFloat("6e-12"));
770 EXPECT_DOUBLE_EQ(1.2 , Tokenizer::ParseFloat("1.2"));
771 EXPECT_DOUBLE_EQ(1.e2 , Tokenizer::ParseFloat("1.e2"));
774 EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1e"));
775 EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1e-"));
776 EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1.e"));
779 EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1f"));
780 EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1.0f"));
781 EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1F"));
785 EXPECT_EQ( 0.0, Tokenizer::ParseFloat("1e-9999999999999999999999999999"));
786 EXPECT_EQ(HUGE_VAL, Tokenizer::ParseFloat("1e+9999999999999999999999999999"));
790 EXPECT_DEBUG_DEATH(Tokenizer::ParseFloat("zxy"),
792 EXPECT_DEBUG_DEATH(Tokenizer::ParseFloat("1-e0"),
794 EXPECT_DEBUG_DEATH(Tokenizer::ParseFloat("-1.0"),
801 Tokenizer::ParseString("'hello'", &output);
803 Tokenizer::ParseString("\"blah\\nblah2\"", &output);
805 Tokenizer::ParseString("'\\1x\\1\\123\\739\\52\\334n\\3'", &output);
807 Tokenizer::ParseString("'\\x20\\x4'", &output);
811 Tokenizer::ParseString("\"\\a\\l\\v\\t", &output); // \l is invalid
813 Tokenizer::ParseString("'", &output);
815 Tokenizer::ParseString("'\\", &output);
820 Tokenizer::ParseString("'\\u0024\\u00a2\\u20ac\\U00024b62XX'", &output);
823 Tokenizer::ParseString("'\\u0024\\u00a2\\u20ac\\ud852\\udf62XX'", &output);
828 Tokenizer::ParseString("'\\ud852XX'", &output);
831 Tokenizer::ParseString("\\u0", &output);
836 EXPECT_DEBUG_DEATH(Tokenizer::ParseString("", &output),
844 Tokenizer::ParseStringAppend("'hello'", &output);
846 Tokenizer::ParseString("'hello'", &output);
856 bool recoverable; // True if the tokenizer should be able to recover and
952 // Set up the tokenizer.
957 Tokenizer tokenizer(&input, &error_collector);
961 while (tokenizer.Next()) {
962 last_was_foo = tokenizer.current().text == "foo";
980 // Create a tokenizer, read one token, then destroy it.
983 Tokenizer tokenizer(&input, &error_collector);
985 tokenizer.Next();
