1 /* 2 * Copyright (C) 2007-2010 Jlio Vilmar Gesser. 3 * Copyright (C) 2011, 2013-2016 The JavaParser Team. 4 * 5 * This file is part of JavaParser. 6 * 7 * JavaParser can be used either under the terms of 8 * a) the GNU Lesser General Public License as published by 9 * the Free Software Foundation, either version 3 of the License, or 10 * (at your option) any later version. 11 * b) the terms of the Apache License 12 * 13 * You should have received a copy of both licenses in LICENCE.LGPL and 14 * LICENCE.APACHE. Please refer to those files for details. 15 * 16 * JavaParser is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 * GNU Lesser General Public License for more details. 20 */ 21 package com.github.javaparser; 22 23 import java.util.List; 24 import java.util.Optional; 25 import static com.github.javaparser.utils.CodeGenerationUtils.f; 26 import static com.github.javaparser.utils.Utils.EOL; 27 import static com.github.javaparser.utils.Utils.assertNotNull; 28 import javax.annotation.Generated; 29 30 /** 31 * A token from a parsed source file. 32 * (Awkwardly named "Java"Token since JavaCC already generates an internal class Token.) 33 * It is a node in a double linked list called token list. 34 */ 35 public class JavaToken { 36 37 public static final JavaToken INVALID = new JavaToken(); 38 39 private Range range; 40 41 private int kind; 42 43 private String text; 44 45 private JavaToken previousToken = null; 46 47 private JavaToken nextToken = null; 48 49 private JavaToken() { 50 this(null, 0, "INVALID", null, null); 51 } 52 53 public JavaToken(int kind, String text) { 54 this(null, kind, text, null, null); 55 } 56 57 JavaToken(Token token, List<JavaToken> tokens) { 58 // You could be puzzled by the following lines 59 // 60 // The reason why these lines are necessary is the fact that Java is ambiguous. There are cases where the 61 // sequence of characters ">>>" and ">>" should be recognized as the single tokens ">>>" and ">>". In other 62 // cases however we want to split those characters in single GT tokens (">"). 63 // 64 // For example, in expressions ">>" and ">>>" are valid, while when defining types we could have this: 65 // 66 // List<List<Set<String>>>> 67 // 68 // You can see that the sequence ">>>>" should be interpreted as four consecutive ">" tokens closing a type 69 // parameter list. 70 // 71 // The JavaCC handle this case by first recognizing always the longest token, and then depending on the context 72 // putting back the unused chars in the stream. However in those cases the token provided is invalid: it has an 73 // image corresponding to the text originally recognized, without considering that after some characters could 74 // have been put back into the stream. 75 // 76 // So in the case of: 77 // 78 // List<List<Set<String>>>> 79 // ___ -> recognized as ">>>", then ">>" put back in the stream but Token(type=GT, image=">>>") passed to this class 80 // ___ -> recognized as ">>>", then ">>" put back in the stream but Token(type=GT, image=">>>") passed to this class 81 // __ -> recognized as ">>", then ">" put back in the stream but Token(type=GT, image=">>") passed to this class 82 // _ -> Token(type=GT, image=">") good! 83 // 84 // So given the image could be wrong but the type is correct, we look at the type of the token and we fix 85 // the image. Everybody is happy and we can keep this horrible thing as our little secret. 86 Range range = Range.range(token.beginLine, token.beginColumn, token.endLine, token.endColumn); 87 String text = token.image; 88 if (token.kind == GeneratedJavaParserConstants.GT) { 89 range = Range.range(token.beginLine, token.beginColumn, token.endLine, token.beginColumn); 90 text = ">"; 91 } else if (token.kind == GeneratedJavaParserConstants.RSIGNEDSHIFT) { 92 range = Range.range(token.beginLine, token.beginColumn, token.endLine, token.beginColumn + 1); 93 text = ">>"; 94 } 95 this.range = range; 96 this.kind = token.kind; 97 this.text = text; 98 if (!tokens.isEmpty()) { 99 final JavaToken previousToken = tokens.get(tokens.size() - 1); 100 this.previousToken = previousToken; 101 previousToken.nextToken = this; 102 } else { 103 previousToken = null; 104 } 105 } 106 107 /** 108 * Create a token of a certain kind. 109 */ 110 public JavaToken(int kind) { 111 String content = GeneratedJavaParserConstants.tokenImage[kind]; 112 if (content.startsWith("\"")) { 113 content = content.substring(1, content.length() - 1); 114 } 115 if (TokenTypes.isEndOfLineToken(kind)) { 116 content = EOL; 117 } else if (TokenTypes.isWhitespace(kind)) { 118 content = " "; 119 } 120 this.kind = kind; 121 this.text = content; 122 } 123 124 public JavaToken(Range range, int kind, String text, JavaToken previousToken, JavaToken nextToken) { 125 assertNotNull(text); 126 this.range = range; 127 this.kind = kind; 128 this.text = text; 129 this.previousToken = previousToken; 130 this.nextToken = nextToken; 131 } 132 133 public Optional<Range> getRange() { 134 return Optional.ofNullable(range); 135 } 136 137 public int getKind() { 138 return kind; 139 } 140 141 void setKind(int kind) { 142 this.kind = kind; 143 } 144 145 public String getText() { 146 return text; 147 } 148 149 public Optional<JavaToken> getNextToken() { 150 return Optional.ofNullable(nextToken); 151 } 152 153 public Optional<JavaToken> getPreviousToken() { 154 return Optional.ofNullable(previousToken); 155 } 156 157 public void setRange(Range range) { 158 this.range = range; 159 } 160 161 public void setText(String text) { 162 this.text = text; 163 } 164 165 public String asString() { 166 return text; 167 } 168 169 /** 170 * @return the token range that goes from the beginning to the end of the token list this token is a part of. 171 */ 172 public TokenRange toTokenRange() { 173 return new TokenRange(findFirstToken(), findLastToken()); 174 } 175 176 @Override 177 public String toString() { 178 String text = getText().replace("\n", "\\n").replace("\r", "\\r").replace("\r\n", "\\r\\n").replace("\t", "\\t"); 179 return f("\"%s\" <%s> %s", text, getKind(), getRange().map(Range::toString).orElse("(?)-(?)")); 180 } 181 182 /** 183 * Used by the parser while constructing nodes. No tokens should be invalid when the parser is done. 184 */ 185 public boolean valid() { 186 return !invalid(); 187 } 188 189 /** 190 * Used by the parser while constructing nodes. No tokens should be invalid when the parser is done. 191 */ 192 public boolean invalid() { 193 return this == INVALID; 194 } 195 196 public enum Category { 197 198 WHITESPACE_NO_EOL, 199 EOL, 200 COMMENT, 201 IDENTIFIER, 202 KEYWORD, 203 LITERAL, 204 SEPARATOR, 205 OPERATOR; 206 207 public boolean isWhitespaceOrComment() { 208 return isWhitespace() || this == COMMENT; 209 } 210 211 public boolean isWhitespace() { 212 return this == WHITESPACE_NO_EOL || this == EOL; 213 } 214 215 public boolean isEndOfLine() { 216 return this == EOL; 217 } 218 219 public boolean isComment() { 220 return this == COMMENT; 221 } 222 223 public boolean isWhitespaceButNotEndOfLine() { 224 return this == WHITESPACE_NO_EOL; 225 } 226 227 public boolean isIdentifier() { 228 return this == IDENTIFIER; 229 } 230 231 public boolean isKeyword() { 232 return this == KEYWORD; 233 } 234 235 public boolean isLiteral() { 236 return this == LITERAL; 237 } 238 239 public boolean isSeparator() { 240 return this == SEPARATOR; 241 } 242 243 public boolean isOperator() { 244 return this == OPERATOR; 245 } 246 } 247 248 @Generated("com.github.javaparser.generator.core.other.TokenKindGenerator") 249 public enum Kind { 250 251 EOF(0), 252 SPACE(1), 253 WINDOWS_EOL(2), 254 UNIX_EOL(3), 255 OLD_MAC_EOL(4), 256 SINGLE_LINE_COMMENT(5), 257 ENTER_JAVADOC_COMMENT(6), 258 ENTER_MULTILINE_COMMENT(7), 259 JAVADOC_COMMENT(8), 260 MULTI_LINE_COMMENT(9), 261 COMMENT_CONTENT(10), 262 ABSTRACT(11), 263 ASSERT(12), 264 BOOLEAN(13), 265 BREAK(14), 266 BYTE(15), 267 CASE(16), 268 CATCH(17), 269 CHAR(18), 270 CLASS(19), 271 CONST(20), 272 CONTINUE(21), 273 _DEFAULT(22), 274 DO(23), 275 DOUBLE(24), 276 ELSE(25), 277 ENUM(26), 278 EXTENDS(27), 279 FALSE(28), 280 FINAL(29), 281 FINALLY(30), 282 FLOAT(31), 283 FOR(32), 284 GOTO(33), 285 IF(34), 286 IMPLEMENTS(35), 287 IMPORT(36), 288 INSTANCEOF(37), 289 INT(38), 290 INTERFACE(39), 291 LONG(40), 292 NATIVE(41), 293 NEW(42), 294 NULL(43), 295 PACKAGE(44), 296 PRIVATE(45), 297 PROTECTED(46), 298 PUBLIC(47), 299 RETURN(48), 300 SHORT(49), 301 STATIC(50), 302 STRICTFP(51), 303 SUPER(52), 304 SWITCH(53), 305 SYNCHRONIZED(54), 306 THIS(55), 307 THROW(56), 308 THROWS(57), 309 TRANSIENT(58), 310 TRUE(59), 311 TRY(60), 312 VOID(61), 313 VOLATILE(62), 314 WHILE(63), 315 REQUIRES(64), 316 TO(65), 317 WITH(66), 318 OPEN(67), 319 OPENS(68), 320 USES(69), 321 MODULE(70), 322 EXPORTS(71), 323 PROVIDES(72), 324 TRANSITIVE(73), 325 LONG_LITERAL(74), 326 INTEGER_LITERAL(75), 327 DECIMAL_LITERAL(76), 328 HEX_LITERAL(77), 329 OCTAL_LITERAL(78), 330 BINARY_LITERAL(79), 331 FLOATING_POINT_LITERAL(80), 332 DECIMAL_FLOATING_POINT_LITERAL(81), 333 DECIMAL_EXPONENT(82), 334 HEXADECIMAL_FLOATING_POINT_LITERAL(83), 335 HEXADECIMAL_EXPONENT(84), 336 HEX_DIGITS(85), 337 UNICODE_ESCAPE(86), 338 CHARACTER_LITERAL(87), 339 STRING_LITERAL(88), 340 IDENTIFIER(89), 341 LETTER(90), 342 PART_LETTER(91), 343 LPAREN(92), 344 RPAREN(93), 345 LBRACE(94), 346 RBRACE(95), 347 LBRACKET(96), 348 RBRACKET(97), 349 SEMICOLON(98), 350 COMMA(99), 351 DOT(100), 352 AT(101), 353 ASSIGN(102), 354 LT(103), 355 BANG(104), 356 TILDE(105), 357 HOOK(106), 358 COLON(107), 359 EQ(108), 360 LE(109), 361 GE(110), 362 NE(111), 363 SC_OR(112), 364 SC_AND(113), 365 INCR(114), 366 DECR(115), 367 PLUS(116), 368 MINUS(117), 369 STAR(118), 370 SLASH(119), 371 BIT_AND(120), 372 BIT_OR(121), 373 XOR(122), 374 REM(123), 375 LSHIFT(124), 376 PLUSASSIGN(125), 377 MINUSASSIGN(126), 378 STARASSIGN(127), 379 SLASHASSIGN(128), 380 ANDASSIGN(129), 381 ORASSIGN(130), 382 XORASSIGN(131), 383 REMASSIGN(132), 384 LSHIFTASSIGN(133), 385 RSIGNEDSHIFTASSIGN(134), 386 RUNSIGNEDSHIFTASSIGN(135), 387 ELLIPSIS(136), 388 ARROW(137), 389 DOUBLECOLON(138), 390 RUNSIGNEDSHIFT(139), 391 RSIGNEDSHIFT(140), 392 GT(141), 393 CTRL_Z(142); 394 395 private final int kind; 396 397 Kind(int kind) { 398 this.kind = kind; 399 } 400 401 public static Kind valueOf(int kind) { 402 switch(kind) { 403 case 142: 404 return CTRL_Z; 405 case 141: 406 return GT; 407 case 140: 408 return RSIGNEDSHIFT; 409 case 139: 410 return RUNSIGNEDSHIFT; 411 case 138: 412 return DOUBLECOLON; 413 case 137: 414 return ARROW; 415 case 136: 416 return ELLIPSIS; 417 case 135: 418 return RUNSIGNEDSHIFTASSIGN; 419 case 134: 420 return RSIGNEDSHIFTASSIGN; 421 case 133: 422 return LSHIFTASSIGN; 423 case 132: 424 return REMASSIGN; 425 case 131: 426 return XORASSIGN; 427 case 130: 428 return ORASSIGN; 429 case 129: 430 return ANDASSIGN; 431 case 128: 432 return SLASHASSIGN; 433 case 127: 434 return STARASSIGN; 435 case 126: 436 return MINUSASSIGN; 437 case 125: 438 return PLUSASSIGN; 439 case 124: 440 return LSHIFT; 441 case 123: 442 return REM; 443 case 122: 444 return XOR; 445 case 121: 446 return BIT_OR; 447 case 120: 448 return BIT_AND; 449 case 119: 450 return SLASH; 451 case 118: 452 return STAR; 453 case 117: 454 return MINUS; 455 case 116: 456 return PLUS; 457 case 115: 458 return DECR; 459 case 114: 460 return INCR; 461 case 113: 462 return SC_AND; 463 case 112: 464 return SC_OR; 465 case 111: 466 return NE; 467 case 110: 468 return GE; 469 case 109: 470 return LE; 471 case 108: 472 return EQ; 473 case 107: 474 return COLON; 475 case 106: 476 return HOOK; 477 case 105: 478 return TILDE; 479 case 104: 480 return BANG; 481 case 103: 482 return LT; 483 case 102: 484 return ASSIGN; 485 case 101: 486 return AT; 487 case 100: 488 return DOT; 489 case 99: 490 return COMMA; 491 case 98: 492 return SEMICOLON; 493 case 97: 494 return RBRACKET; 495 case 96: 496 return LBRACKET; 497 case 95: 498 return RBRACE; 499 case 94: 500 return LBRACE; 501 case 93: 502 return RPAREN; 503 case 92: 504 return LPAREN; 505 case 91: 506 return PART_LETTER; 507 case 90: 508 return LETTER; 509 case 89: 510 return IDENTIFIER; 511 case 88: 512 return STRING_LITERAL; 513 case 87: 514 return CHARACTER_LITERAL; 515 case 86: 516 return UNICODE_ESCAPE; 517 case 85: 518 return HEX_DIGITS; 519 case 84: 520 return HEXADECIMAL_EXPONENT; 521 case 83: 522 return HEXADECIMAL_FLOATING_POINT_LITERAL; 523 case 82: 524 return DECIMAL_EXPONENT; 525 case 81: 526 return DECIMAL_FLOATING_POINT_LITERAL; 527 case 80: 528 return FLOATING_POINT_LITERAL; 529 case 79: 530 return BINARY_LITERAL; 531 case 78: 532 return OCTAL_LITERAL; 533 case 77: 534 return HEX_LITERAL; 535 case 76: 536 return DECIMAL_LITERAL; 537 case 75: 538 return INTEGER_LITERAL; 539 case 74: 540 return LONG_LITERAL; 541 case 73: 542 return TRANSITIVE; 543 case 72: 544 return PROVIDES; 545 case 71: 546 return EXPORTS; 547 case 70: 548 return MODULE; 549 case 69: 550 return USES; 551 case 68: 552 return OPENS; 553 case 67: 554 return OPEN; 555 case 66: 556 return WITH; 557 case 65: 558 return TO; 559 case 64: 560 return REQUIRES; 561 case 63: 562 return WHILE; 563 case 62: 564 return VOLATILE; 565 case 61: 566 return VOID; 567 case 60: 568 return TRY; 569 case 59: 570 return TRUE; 571 case 58: 572 return TRANSIENT; 573 case 57: 574 return THROWS; 575 case 56: 576 return THROW; 577 case 55: 578 return THIS; 579 case 54: 580 return SYNCHRONIZED; 581 case 53: 582 return SWITCH; 583 case 52: 584 return SUPER; 585 case 51: 586 return STRICTFP; 587 case 50: 588 return STATIC; 589 case 49: 590 return SHORT; 591 case 48: 592 return RETURN; 593 case 47: 594 return PUBLIC; 595 case 46: 596 return PROTECTED; 597 case 45: 598 return PRIVATE; 599 case 44: 600 return PACKAGE; 601 case 43: 602 return NULL; 603 case 42: 604 return NEW; 605 case 41: 606 return NATIVE; 607 case 40: 608 return LONG; 609 case 39: 610 return INTERFACE; 611 case 38: 612 return INT; 613 case 37: 614 return INSTANCEOF; 615 case 36: 616 return IMPORT; 617 case 35: 618 return IMPLEMENTS; 619 case 34: 620 return IF; 621 case 33: 622 return GOTO; 623 case 32: 624 return FOR; 625 case 31: 626 return FLOAT; 627 case 30: 628 return FINALLY; 629 case 29: 630 return FINAL; 631 case 28: 632 return FALSE; 633 case 27: 634 return EXTENDS; 635 case 26: 636 return ENUM; 637 case 25: 638 return ELSE; 639 case 24: 640 return DOUBLE; 641 case 23: 642 return DO; 643 case 22: 644 return _DEFAULT; 645 case 21: 646 return CONTINUE; 647 case 20: 648 return CONST; 649 case 19: 650 return CLASS; 651 case 18: 652 return CHAR; 653 case 17: 654 return CATCH; 655 case 16: 656 return CASE; 657 case 15: 658 return BYTE; 659 case 14: 660 return BREAK; 661 case 13: 662 return BOOLEAN; 663 case 12: 664 return ASSERT; 665 case 11: 666 return ABSTRACT; 667 case 10: 668 return COMMENT_CONTENT; 669 case 9: 670 return MULTI_LINE_COMMENT; 671 case 8: 672 return JAVADOC_COMMENT; 673 case 7: 674 return ENTER_MULTILINE_COMMENT; 675 case 6: 676 return ENTER_JAVADOC_COMMENT; 677 case 5: 678 return SINGLE_LINE_COMMENT; 679 case 4: 680 return OLD_MAC_EOL; 681 case 3: 682 return UNIX_EOL; 683 case 2: 684 return WINDOWS_EOL; 685 case 1: 686 return SPACE; 687 case 0: 688 return EOF; 689 default: 690 throw new IllegalArgumentException(f("Token kind %i is unknown.", kind)); 691 } 692 } 693 694 public int getKind() { 695 return kind; 696 } 697 } 698 699 public JavaToken.Category getCategory() { 700 return TokenTypes.getCategory(kind); 701 } 702 703 /** 704 * Inserts newToken into the token list just before this token. 705 */ 706 public void insert(JavaToken newToken) { 707 assertNotNull(newToken); 708 getPreviousToken().ifPresent(p -> { 709 p.nextToken = newToken; 710 newToken.previousToken = p; 711 }); 712 previousToken = newToken; 713 newToken.nextToken = this; 714 } 715 716 /** 717 * Inserts newToken into the token list just after this token. 718 */ 719 public void insertAfter(JavaToken newToken) { 720 assertNotNull(newToken); 721 getNextToken().ifPresent(n -> { 722 n.previousToken = newToken; 723 newToken.nextToken = n; 724 }); 725 nextToken = newToken; 726 newToken.previousToken = this; 727 } 728 729 /** 730 * Links the tokens around the current token together, making the current token disappear from the list. 731 */ 732 public void deleteToken() { 733 final Optional<JavaToken> nextToken = getNextToken(); 734 final Optional<JavaToken> previousToken = getPreviousToken(); 735 previousToken.ifPresent(p -> p.nextToken = nextToken.orElse(null)); 736 nextToken.ifPresent(n -> n.previousToken = previousToken.orElse(null)); 737 } 738 739 /** 740 * Replaces the current token with newToken. 741 */ 742 public void replaceToken(JavaToken newToken) { 743 assertNotNull(newToken); 744 getPreviousToken().ifPresent(p -> { 745 p.nextToken = newToken; 746 newToken.previousToken = p; 747 }); 748 getNextToken().ifPresent(n -> { 749 n.previousToken = newToken; 750 newToken.nextToken = n; 751 }); 752 } 753 754 /** 755 * @return the last token in the token list. 756 */ 757 public JavaToken findLastToken() { 758 JavaToken current = this; 759 while (current.getNextToken().isPresent()) { 760 current = current.getNextToken().get(); 761 } 762 return current; 763 } 764 765 /** 766 * @return the first token in the token list. 767 */ 768 public JavaToken findFirstToken() { 769 JavaToken current = this; 770 while (current.getPreviousToken().isPresent()) { 771 current = current.getPreviousToken().get(); 772 } 773 return current; 774 } 775 776 @Override 777 public int hashCode() { 778 int result = kind; 779 result = 31 * result + text.hashCode(); 780 return result; 781 } 782 783 @Override 784 public boolean equals(Object o) { 785 if (this == o) 786 return true; 787 if (o == null || getClass() != o.getClass()) 788 return false; 789 JavaToken javaToken = (JavaToken) o; 790 if (kind != javaToken.kind) 791 return false; 792 if (!text.equals(javaToken.text)) 793 return false; 794 return true; 795 } 796 } 797
