1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include <iostream> 6 #include <sstream> 7 8 #include "testing/gtest/include/gtest/gtest.h" 9 #include "tools/gn/input_file.h" 10 #include "tools/gn/parser.h" 11 #include "tools/gn/tokenizer.h" 12 13 namespace { 14 15 bool GetTokens(const InputFile* input, std::vector<Token>* result) { 16 result->clear(); 17 Err err; 18 *result = Tokenizer::Tokenize(input, &err); 19 return !err.has_error(); 20 } 21 22 void DoParserPrintTest(const char* input, const char* expected) { 23 std::vector<Token> tokens; 24 InputFile input_file(SourceFile("/test")); 25 input_file.SetContents(input); 26 ASSERT_TRUE(GetTokens(&input_file, &tokens)); 27 28 Err err; 29 scoped_ptr<ParseNode> result = Parser::Parse(tokens, &err); 30 if (!result) 31 err.PrintToStdout(); 32 ASSERT_TRUE(result); 33 34 std::ostringstream collector; 35 result->Print(collector, 0); 36 37 EXPECT_EQ(expected, collector.str()); 38 } 39 40 void DoExpressionPrintTest(const char* input, const char* expected) { 41 std::vector<Token> tokens; 42 InputFile input_file(SourceFile("/test")); 43 input_file.SetContents(input); 44 ASSERT_TRUE(GetTokens(&input_file, &tokens)); 45 46 Err err; 47 scoped_ptr<ParseNode> result = Parser::ParseExpression(tokens, &err); 48 ASSERT_TRUE(result); 49 50 std::ostringstream collector; 51 result->Print(collector, 0); 52 53 EXPECT_EQ(expected, collector.str()); 54 } 55 56 // Expects the tokenizer or parser to identify an error at the given line and 57 // character. 58 void DoParserErrorTest(const char* input, int err_line, int err_char) { 59 InputFile input_file(SourceFile("/test")); 60 input_file.SetContents(input); 61 62 Err err; 63 std::vector<Token> tokens = Tokenizer::Tokenize(&input_file, &err); 64 if (!err.has_error()) { 65 scoped_ptr<ParseNode> result = Parser::Parse(tokens, &err); 66 ASSERT_FALSE(result); 67 ASSERT_TRUE(err.has_error()); 68 } 69 70 EXPECT_EQ(err_line, err.location().line_number()); 71 EXPECT_EQ(err_char, err.location().char_offset()); 72 } 73 74 // Expects the tokenizer or parser to identify an error at the given line and 75 // character. 76 void DoExpressionErrorTest(const char* input, int err_line, int err_char) { 77 InputFile input_file(SourceFile("/test")); 78 input_file.SetContents(input); 79 80 Err err; 81 std::vector<Token> tokens = Tokenizer::Tokenize(&input_file, &err); 82 if (!err.has_error()) { 83 scoped_ptr<ParseNode> result = Parser::ParseExpression(tokens, &err); 84 ASSERT_FALSE(result); 85 ASSERT_TRUE(err.has_error()); 86 } 87 88 EXPECT_EQ(err_line, err.location().line_number()); 89 EXPECT_EQ(err_char, err.location().char_offset()); 90 } 91 92 } // namespace 93 94 TEST(Parser, Literal) { 95 DoExpressionPrintTest("5", "LITERAL(5)\n"); 96 DoExpressionPrintTest("\"stuff\"", "LITERAL(\"stuff\")\n"); 97 } 98 99 TEST(Parser, BinaryOp) { 100 // TODO(scottmg): The tokenizer is dumb, and treats "5-1" as two integers, 101 // not a binary operator between two positive integers. 102 DoExpressionPrintTest("5 - 1", 103 "BINARY(-)\n" 104 " LITERAL(5)\n" 105 " LITERAL(1)\n"); 106 DoExpressionPrintTest("5+1", 107 "BINARY(+)\n" 108 " LITERAL(5)\n" 109 " LITERAL(1)\n"); 110 DoExpressionPrintTest("5 - 1 - 2", 111 "BINARY(-)\n" 112 " BINARY(-)\n" 113 " LITERAL(5)\n" 114 " LITERAL(1)\n" 115 " LITERAL(2)\n"); 116 } 117 118 TEST(Parser, FunctionCall) { 119 DoExpressionPrintTest("foo()", 120 "FUNCTION(foo)\n" 121 " LIST\n"); 122 DoExpressionPrintTest("blah(1, 2)", 123 "FUNCTION(blah)\n" 124 " LIST\n" 125 " LITERAL(1)\n" 126 " LITERAL(2)\n"); 127 DoExpressionErrorTest("foo(1, 2,)", 1, 10); 128 DoExpressionErrorTest("foo(1 2)", 1, 7); 129 } 130 131 TEST(Parser, ParenExpression) { 132 const char* input = "(foo(1)) + (a + (b - c) + d)"; 133 const char* expected = 134 "BINARY(+)\n" 135 " FUNCTION(foo)\n" 136 " LIST\n" 137 " LITERAL(1)\n" 138 " BINARY(+)\n" 139 " BINARY(+)\n" 140 " IDENTIFIER(a)\n" 141 " BINARY(-)\n" 142 " IDENTIFIER(b)\n" 143 " IDENTIFIER(c)\n" 144 " IDENTIFIER(d)\n"; 145 DoExpressionPrintTest(input, expected); 146 DoExpressionErrorTest("(a +", 1, 4); 147 } 148 149 TEST(Parser, OrderOfOperationsLeftAssociative) { 150 const char* input = "5 - 1 - 2\n"; 151 const char* expected = 152 "BINARY(-)\n" 153 " BINARY(-)\n" 154 " LITERAL(5)\n" 155 " LITERAL(1)\n" 156 " LITERAL(2)\n"; 157 DoExpressionPrintTest(input, expected); 158 } 159 160 TEST(Parser, OrderOfOperationsEqualityBoolean) { 161 const char* input = 162 "if (a == \"b\" && is_stuff) {\n" 163 " print(\"hai\")\n" 164 "}\n"; 165 const char* expected = 166 "BLOCK\n" 167 " CONDITION\n" 168 " BINARY(&&)\n" 169 " BINARY(==)\n" 170 " IDENTIFIER(a)\n" 171 " LITERAL(\"b\")\n" 172 " IDENTIFIER(is_stuff)\n" 173 " BLOCK\n" 174 " FUNCTION(print)\n" 175 " LIST\n" 176 " LITERAL(\"hai\")\n"; 177 DoParserPrintTest(input, expected); 178 } 179 180 TEST(Parser, UnaryOp) { 181 DoExpressionPrintTest("!foo", 182 "UNARY(!)\n" 183 " IDENTIFIER(foo)\n"); 184 } 185 186 TEST(Parser, List) { 187 DoExpressionPrintTest("[]", "LIST\n"); 188 DoExpressionPrintTest("[1,asd,]", 189 "LIST\n" 190 " LITERAL(1)\n" 191 " IDENTIFIER(asd)\n"); 192 DoExpressionPrintTest("[1, 2+3 - foo]", 193 "LIST\n" 194 " LITERAL(1)\n" 195 " BINARY(-)\n" 196 " BINARY(+)\n" 197 " LITERAL(2)\n" 198 " LITERAL(3)\n" 199 " IDENTIFIER(foo)\n"); 200 DoExpressionPrintTest("[1,\n2,\n 3,\n 4]", 201 "LIST\n" 202 " LITERAL(1)\n" 203 " LITERAL(2)\n" 204 " LITERAL(3)\n" 205 " LITERAL(4)\n"); 206 207 DoExpressionErrorTest("[a, 2+,]", 1, 6); 208 DoExpressionErrorTest("[,]", 1, 2); 209 DoExpressionErrorTest("[a,,]", 1, 4); 210 } 211 212 TEST(Parser, Assignment) { 213 DoParserPrintTest("a=2", 214 "BLOCK\n" 215 " BINARY(=)\n" 216 " IDENTIFIER(a)\n" 217 " LITERAL(2)\n"); 218 } 219 220 TEST(Parser, Accessor) { 221 // Accessor indexing. 222 DoParserPrintTest("a=b[c+2]", 223 "BLOCK\n" 224 " BINARY(=)\n" 225 " IDENTIFIER(a)\n" 226 " ACCESSOR\n" 227 " b\n" // AccessorNode is a bit weird in that it holds 228 // a Token, not a ParseNode for the base. 229 " BINARY(+)\n" 230 " IDENTIFIER(c)\n" 231 " LITERAL(2)\n"); 232 DoParserErrorTest("a = b[1][0]", 1, 5); 233 234 // Member accessors. 235 DoParserPrintTest("a=b.c+2", 236 "BLOCK\n" 237 " BINARY(=)\n" 238 " IDENTIFIER(a)\n" 239 " BINARY(+)\n" 240 " ACCESSOR\n" 241 " b\n" 242 " IDENTIFIER(c)\n" 243 " LITERAL(2)\n"); 244 DoParserErrorTest("a = b.c.d", 1, 6); // Can't nest accessors (currently). 245 DoParserErrorTest("a.b = 5", 1, 1); // Can't assign to accessors (currently). 246 } 247 248 TEST(Parser, Condition) { 249 DoParserPrintTest("if(1) { a = 2 }", 250 "BLOCK\n" 251 " CONDITION\n" 252 " LITERAL(1)\n" 253 " BLOCK\n" 254 " BINARY(=)\n" 255 " IDENTIFIER(a)\n" 256 " LITERAL(2)\n"); 257 258 DoParserPrintTest("if(1) { a = 2 } else if (0) { a = 3 } else { a = 4 }", 259 "BLOCK\n" 260 " CONDITION\n" 261 " LITERAL(1)\n" 262 " BLOCK\n" 263 " BINARY(=)\n" 264 " IDENTIFIER(a)\n" 265 " LITERAL(2)\n" 266 " CONDITION\n" 267 " LITERAL(0)\n" 268 " BLOCK\n" 269 " BINARY(=)\n" 270 " IDENTIFIER(a)\n" 271 " LITERAL(3)\n" 272 " BLOCK\n" 273 " BINARY(=)\n" 274 " IDENTIFIER(a)\n" 275 " LITERAL(4)\n"); 276 } 277 278 TEST(Parser, OnlyCallAndAssignInBody) { 279 DoParserErrorTest("[]", 1, 2); 280 DoParserErrorTest("3 + 4", 1, 5); 281 DoParserErrorTest("6 - 7", 1, 5); 282 DoParserErrorTest("if (1) { 5 } else { print(4) }", 1, 12); 283 } 284 285 TEST(Parser, NoAssignmentInCondition) { 286 DoParserErrorTest("if (a=2) {}", 1, 5); 287 } 288 289 TEST(Parser, CompleteFunction) { 290 const char* input = 291 "cc_test(\"foo\") {\n" 292 " sources = [\n" 293 " \"foo.cc\",\n" 294 " \"foo.h\"\n" 295 " ]\n" 296 " dependencies = [\n" 297 " \"base\"\n" 298 " ]\n" 299 "}\n"; 300 const char* expected = 301 "BLOCK\n" 302 " FUNCTION(cc_test)\n" 303 " LIST\n" 304 " LITERAL(\"foo\")\n" 305 " BLOCK\n" 306 " BINARY(=)\n" 307 " IDENTIFIER(sources)\n" 308 " LIST\n" 309 " LITERAL(\"foo.cc\")\n" 310 " LITERAL(\"foo.h\")\n" 311 " BINARY(=)\n" 312 " IDENTIFIER(dependencies)\n" 313 " LIST\n" 314 " LITERAL(\"base\")\n"; 315 DoParserPrintTest(input, expected); 316 } 317 318 TEST(Parser, FunctionWithConditional) { 319 const char* input = 320 "cc_test(\"foo\") {\n" 321 " sources = [\"foo.cc\"]\n" 322 " if (OS == \"mac\") {\n" 323 " sources += \"bar.cc\"\n" 324 " } else if (OS == \"win\") {\n" 325 " sources -= [\"asd.cc\", \"foo.cc\"]\n" 326 " } else {\n" 327 " dependencies += [\"bar.cc\"]\n" 328 " }\n" 329 "}\n"; 330 const char* expected = 331 "BLOCK\n" 332 " FUNCTION(cc_test)\n" 333 " LIST\n" 334 " LITERAL(\"foo\")\n" 335 " BLOCK\n" 336 " BINARY(=)\n" 337 " IDENTIFIER(sources)\n" 338 " LIST\n" 339 " LITERAL(\"foo.cc\")\n" 340 " CONDITION\n" 341 " BINARY(==)\n" 342 " IDENTIFIER(OS)\n" 343 " LITERAL(\"mac\")\n" 344 " BLOCK\n" 345 " BINARY(+=)\n" 346 " IDENTIFIER(sources)\n" 347 " LITERAL(\"bar.cc\")\n" 348 " CONDITION\n" 349 " BINARY(==)\n" 350 " IDENTIFIER(OS)\n" 351 " LITERAL(\"win\")\n" 352 " BLOCK\n" 353 " BINARY(-=)\n" 354 " IDENTIFIER(sources)\n" 355 " LIST\n" 356 " LITERAL(\"asd.cc\")\n" 357 " LITERAL(\"foo.cc\")\n" 358 " BLOCK\n" 359 " BINARY(+=)\n" 360 " IDENTIFIER(dependencies)\n" 361 " LIST\n" 362 " LITERAL(\"bar.cc\")\n"; 363 DoParserPrintTest(input, expected); 364 } 365 366 TEST(Parser, NestedBlocks) { 367 const char* input = "{cc_test(\"foo\") {{foo=1}\n{}}}"; 368 const char* expected = 369 "BLOCK\n" 370 " BLOCK\n" 371 " FUNCTION(cc_test)\n" 372 " LIST\n" 373 " LITERAL(\"foo\")\n" 374 " BLOCK\n" 375 " BLOCK\n" 376 " BINARY(=)\n" 377 " IDENTIFIER(foo)\n" 378 " LITERAL(1)\n" 379 " BLOCK\n"; 380 DoParserPrintTest(input, expected); 381 const char* input_with_newline = "{cc_test(\"foo\") {{foo=1}\n{}}}"; 382 DoParserPrintTest(input_with_newline, expected); 383 } 384 385 TEST(Parser, UnterminatedBlock) { 386 DoParserErrorTest("stuff() {", 1, 9); 387 } 388 389 TEST(Parser, BadlyTerminatedNumber) { 390 DoParserErrorTest("1234z", 1, 5); 391 } 392 393 TEST(Parser, NewlinesInUnusualPlaces) { 394 DoParserPrintTest( 395 "if\n" 396 "(\n" 397 "a\n" 398 ")\n" 399 "{\n" 400 "}\n", 401 "BLOCK\n" 402 " CONDITION\n" 403 " IDENTIFIER(a)\n" 404 " BLOCK\n"); 405 } 406 407 TEST(Parser, NewlinesInUnusualPlaces2) { 408 DoParserPrintTest( 409 "a\n=\n2\n", 410 "BLOCK\n" 411 " BINARY(=)\n" 412 " IDENTIFIER(a)\n" 413 " LITERAL(2)\n"); 414 DoParserPrintTest( 415 "x =\ny if\n(1\n) {}", 416 "BLOCK\n" 417 " BINARY(=)\n" 418 " IDENTIFIER(x)\n" 419 " IDENTIFIER(y)\n" 420 " CONDITION\n" 421 " LITERAL(1)\n" 422 " BLOCK\n"); 423 DoParserPrintTest( 424 "x = 3\n+2", 425 "BLOCK\n" 426 " BINARY(=)\n" 427 " IDENTIFIER(x)\n" 428 " BINARY(+)\n" 429 " LITERAL(3)\n" 430 " LITERAL(2)\n" 431 ); 432 } 433 434 TEST(Parser, NewlineBeforeSubscript) { 435 const char* input = "a = b[1]"; 436 const char* input_with_newline = "a = b\n[1]"; 437 const char* expected = 438 "BLOCK\n" 439 " BINARY(=)\n" 440 " IDENTIFIER(a)\n" 441 " ACCESSOR\n" 442 " b\n" 443 " LITERAL(1)\n"; 444 DoParserPrintTest( 445 input, 446 expected); 447 DoParserPrintTest( 448 input_with_newline, 449 expected); 450 } 451 452 TEST(Parser, SequenceOfExpressions) { 453 DoParserPrintTest( 454 "a = 1 b = 2", 455 "BLOCK\n" 456 " BINARY(=)\n" 457 " IDENTIFIER(a)\n" 458 " LITERAL(1)\n" 459 " BINARY(=)\n" 460 " IDENTIFIER(b)\n" 461 " LITERAL(2)\n"); 462 } 463 464 TEST(Parser, BlockAfterFunction) { 465 const char* input = "func(\"stuff\") {\n}"; 466 // TODO(scottmg): Do we really want these to mean different things? 467 const char* input_with_newline = "func(\"stuff\")\n{\n}"; 468 const char* expected = 469 "BLOCK\n" 470 " FUNCTION(func)\n" 471 " LIST\n" 472 " LITERAL(\"stuff\")\n" 473 " BLOCK\n"; 474 DoParserPrintTest(input, expected); 475 DoParserPrintTest(input_with_newline, expected); 476 } 477 478 TEST(Parser, LongExpression) { 479 const char* input = "a = b + c && d || e"; 480 const char* expected = 481 "BLOCK\n" 482 " BINARY(=)\n" 483 " IDENTIFIER(a)\n" 484 " BINARY(||)\n" 485 " BINARY(&&)\n" 486 " BINARY(+)\n" 487 " IDENTIFIER(b)\n" 488 " IDENTIFIER(c)\n" 489 " IDENTIFIER(d)\n" 490 " IDENTIFIER(e)\n"; 491 DoParserPrintTest(input, expected); 492 } 493 494 TEST(Parser, CommentsStandalone) { 495 const char* input = 496 "# Toplevel comment.\n" 497 "\n" 498 "executable(\"wee\") {}\n"; 499 const char* expected = 500 "BLOCK\n" 501 " BLOCK_COMMENT(# Toplevel comment.)\n" 502 " FUNCTION(executable)\n" 503 " LIST\n" 504 " LITERAL(\"wee\")\n" 505 " BLOCK\n"; 506 DoParserPrintTest(input, expected); 507 } 508 509 TEST(Parser, CommentsStandaloneEof) { 510 const char* input = 511 "executable(\"wee\") {}\n" 512 "# EOF comment.\n"; 513 const char* expected = 514 "BLOCK\n" 515 " +AFTER_COMMENT(\"# EOF comment.\")\n" 516 " FUNCTION(executable)\n" 517 " LIST\n" 518 " LITERAL(\"wee\")\n" 519 " BLOCK\n"; 520 DoParserPrintTest(input, expected); 521 } 522 523 TEST(Parser, CommentsLineAttached) { 524 const char* input = 525 "executable(\"wee\") {\n" 526 " # Some sources.\n" 527 " sources = [\n" 528 " \"stuff.cc\",\n" 529 " \"things.cc\",\n" 530 " # This file is special or something.\n" 531 " \"another.cc\",\n" 532 " ]\n" 533 "}\n"; 534 const char* expected = 535 "BLOCK\n" 536 " FUNCTION(executable)\n" 537 " LIST\n" 538 " LITERAL(\"wee\")\n" 539 " BLOCK\n" 540 " BINARY(=)\n" 541 " +BEFORE_COMMENT(\"# Some sources.\")\n" 542 " IDENTIFIER(sources)\n" 543 " LIST\n" 544 " LITERAL(\"stuff.cc\")\n" 545 " LITERAL(\"things.cc\")\n" 546 " LITERAL(\"another.cc\")\n" 547 " +BEFORE_COMMENT(\"# This file is special or something.\")\n"; 548 DoParserPrintTest(input, expected); 549 } 550 551 TEST(Parser, CommentsSuffix) { 552 const char* input = 553 "executable(\"wee\") { # This is some stuff.\n" 554 "sources = [ \"a.cc\" # And another comment here.\n" 555 "] }"; 556 const char* expected = 557 "BLOCK\n" 558 " FUNCTION(executable)\n" 559 " LIST\n" 560 " LITERAL(\"wee\")\n" 561 " +SUFFIX_COMMENT(\"# This is some stuff.\")\n" 562 " BLOCK\n" 563 " BINARY(=)\n" 564 " IDENTIFIER(sources)\n" 565 " LIST\n" 566 " LITERAL(\"a.cc\")\n" 567 " +SUFFIX_COMMENT(\"# And another comment here.\")\n"; 568 DoParserPrintTest(input, expected); 569 } 570 571 TEST(Parser, CommentsSuffixDifferentLine) { 572 const char* input = 573 "executable(\"wee\") {\n" 574 " sources = [ \"a\",\n" 575 " \"b\" ] # Comment\n" 576 "}\n"; 577 const char* expected = 578 "BLOCK\n" 579 " FUNCTION(executable)\n" 580 " LIST\n" 581 " LITERAL(\"wee\")\n" 582 " BLOCK\n" 583 " BINARY(=)\n" 584 " IDENTIFIER(sources)\n" 585 " LIST\n" 586 " LITERAL(\"a\")\n" 587 " LITERAL(\"b\")\n" 588 " +SUFFIX_COMMENT(\"# Comment\")\n"; 589 DoParserPrintTest(input, expected); 590 } 591 592 TEST(Parser, CommentsSuffixMultiple) { 593 const char* input = 594 "executable(\"wee\") {\n" 595 " sources = [\n" 596 " \"a\", # This is a comment,\n" 597 " # and some more,\n" // Note that this is aligned with above. 598 " # then the end.\n" 599 " ]\n" 600 "}\n"; 601 const char* expected = 602 "BLOCK\n" 603 " FUNCTION(executable)\n" 604 " LIST\n" 605 " LITERAL(\"wee\")\n" 606 " BLOCK\n" 607 " BINARY(=)\n" 608 " IDENTIFIER(sources)\n" 609 " LIST\n" 610 " LITERAL(\"a\")\n" 611 " +SUFFIX_COMMENT(\"# This is a comment,\")\n" 612 " +SUFFIX_COMMENT(\"# and some more,\")\n" 613 " +SUFFIX_COMMENT(\"# then the end.\")\n"; 614 DoParserPrintTest(input, expected); 615 } 616 617 TEST(Parser, CommentsConnectedInList) { 618 const char* input = 619 "defines = [\n" 620 "\n" 621 " # Connected comment.\n" 622 " \"WEE\",\n" 623 " \"BLORPY\",\n" 624 "]\n"; 625 const char* expected = 626 "BLOCK\n" 627 " BINARY(=)\n" 628 " IDENTIFIER(defines)\n" 629 " LIST\n" 630 " LITERAL(\"WEE\")\n" 631 " +BEFORE_COMMENT(\"# Connected comment.\")\n" 632 " LITERAL(\"BLORPY\")\n"; 633 DoParserPrintTest(input, expected); 634 } 635 636 TEST(Parser, HangingIf) { 637 DoParserErrorTest("if", 1, 1); 638 } 639 640 TEST(Parser, NegatingList) { 641 DoParserErrorTest("executable(\"wee\") { sources =- [ \"foo.cc\" ] }", 1, 30); 642 } 643
