Home | History | Annotate | Download | only in AST
      1 //===- unittests/AST/CommentLexer.cpp ------ Comment lexer tests ----------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 
     10 #include "clang/AST/CommentLexer.h"
     11 #include "clang/AST/CommentCommandTraits.h"
     12 #include "clang/Basic/CommentOptions.h"
     13 #include "clang/Basic/Diagnostic.h"
     14 #include "clang/Basic/DiagnosticOptions.h"
     15 #include "clang/Basic/FileManager.h"
     16 #include "clang/Basic/SourceManager.h"
     17 #include "llvm/ADT/STLExtras.h"
     18 #include "gtest/gtest.h"
     19 #include <vector>
     20 
     21 using namespace llvm;
     22 using namespace clang;
     23 
     24 namespace clang {
     25 namespace comments {
     26 
     27 namespace {
     28 class CommentLexerTest : public ::testing::Test {
     29 protected:
     30   CommentLexerTest()
     31     : FileMgr(FileMgrOpts),
     32       DiagID(new DiagnosticIDs()),
     33       Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
     34       SourceMgr(Diags, FileMgr),
     35       Traits(Allocator, CommentOptions()) {
     36   }
     37 
     38   FileSystemOptions FileMgrOpts;
     39   FileManager FileMgr;
     40   IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
     41   DiagnosticsEngine Diags;
     42   SourceManager SourceMgr;
     43   llvm::BumpPtrAllocator Allocator;
     44   CommandTraits Traits;
     45 
     46   void lexString(const char *Source, std::vector<Token> &Toks);
     47 
     48   StringRef getCommandName(const Token &Tok) {
     49     return Traits.getCommandInfo(Tok.getCommandID())->Name;
     50   }
     51 
     52   StringRef getVerbatimBlockName(const Token &Tok) {
     53     return Traits.getCommandInfo(Tok.getVerbatimBlockID())->Name;
     54   }
     55 
     56   StringRef getVerbatimLineName(const Token &Tok) {
     57     return Traits.getCommandInfo(Tok.getVerbatimLineID())->Name;
     58   }
     59 };
     60 
     61 void CommentLexerTest::lexString(const char *Source,
     62                                  std::vector<Token> &Toks) {
     63   MemoryBuffer *Buf = MemoryBuffer::getMemBuffer(Source);
     64   FileID File = SourceMgr.createFileIDForMemBuffer(Buf);
     65   SourceLocation Begin = SourceMgr.getLocForStartOfFile(File);
     66 
     67   Lexer L(Allocator, Diags, Traits, Begin, Source, Source + strlen(Source));
     68 
     69   while (1) {
     70     Token Tok;
     71     L.lex(Tok);
     72     if (Tok.is(tok::eof))
     73       break;
     74     Toks.push_back(Tok);
     75   }
     76 }
     77 
     78 } // unnamed namespace
     79 
     80 // Empty source range should be handled.
     81 TEST_F(CommentLexerTest, Basic1) {
     82   const char *Source = "";
     83   std::vector<Token> Toks;
     84 
     85   lexString(Source, Toks);
     86 
     87   ASSERT_EQ(0U, Toks.size());
     88 }
     89 
     90 // Empty comments should be handled.
     91 TEST_F(CommentLexerTest, Basic2) {
     92   const char *Sources[] = {
     93     "//", "///", "//!", "///<", "//!<"
     94   };
     95   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
     96     std::vector<Token> Toks;
     97 
     98     lexString(Sources[i], Toks);
     99 
    100     ASSERT_EQ(1U, Toks.size());
    101 
    102     ASSERT_EQ(tok::newline, Toks[0].getKind());
    103   }
    104 }
    105 
    106 // Empty comments should be handled.
    107 TEST_F(CommentLexerTest, Basic3) {
    108   const char *Sources[] = {
    109     "/**/", "/***/", "/*!*/", "/**<*/", "/*!<*/"
    110   };
    111   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
    112     std::vector<Token> Toks;
    113 
    114     lexString(Sources[i], Toks);
    115 
    116     ASSERT_EQ(2U, Toks.size());
    117 
    118     ASSERT_EQ(tok::newline, Toks[0].getKind());
    119     ASSERT_EQ(tok::newline, Toks[1].getKind());
    120   }
    121 }
    122 
    123 // Single comment with plain text.
    124 TEST_F(CommentLexerTest, Basic4) {
    125   const char *Sources[] = {
    126     "// Meow",   "/// Meow",    "//! Meow",
    127     "// Meow\n", "// Meow\r\n", "//! Meow\r",
    128   };
    129 
    130   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
    131     std::vector<Token> Toks;
    132 
    133     lexString(Sources[i], Toks);
    134 
    135     ASSERT_EQ(2U, Toks.size());
    136 
    137     ASSERT_EQ(tok::text,          Toks[0].getKind());
    138     ASSERT_EQ(StringRef(" Meow"), Toks[0].getText());
    139 
    140     ASSERT_EQ(tok::newline,       Toks[1].getKind());
    141   }
    142 }
    143 
    144 // Single comment with plain text.
    145 TEST_F(CommentLexerTest, Basic5) {
    146   const char *Sources[] = {
    147     "/* Meow*/", "/** Meow*/",  "/*! Meow*/"
    148   };
    149 
    150   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
    151     std::vector<Token> Toks;
    152 
    153     lexString(Sources[i], Toks);
    154 
    155     ASSERT_EQ(3U, Toks.size());
    156 
    157     ASSERT_EQ(tok::text,          Toks[0].getKind());
    158     ASSERT_EQ(StringRef(" Meow"), Toks[0].getText());
    159 
    160     ASSERT_EQ(tok::newline,       Toks[1].getKind());
    161     ASSERT_EQ(tok::newline,       Toks[2].getKind());
    162   }
    163 }
    164 
    165 // Test newline escaping.
    166 TEST_F(CommentLexerTest, Basic6) {
    167   const char *Sources[] = {
    168     "// Aaa\\\n"   " Bbb\\ \n"   " Ccc?" "?/\n",
    169     "// Aaa\\\r\n" " Bbb\\ \r\n" " Ccc?" "?/\r\n",
    170     "// Aaa\\\r"   " Bbb\\ \r"   " Ccc?" "?/\r"
    171   };
    172 
    173   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
    174     std::vector<Token> Toks;
    175 
    176     lexString(Sources[i], Toks);
    177 
    178     ASSERT_EQ(10U, Toks.size());
    179 
    180     ASSERT_EQ(tok::text,         Toks[0].getKind());
    181     ASSERT_EQ(StringRef(" Aaa"), Toks[0].getText());
    182     ASSERT_EQ(tok::text,         Toks[1].getKind());
    183     ASSERT_EQ(StringRef("\\"),   Toks[1].getText());
    184     ASSERT_EQ(tok::newline,      Toks[2].getKind());
    185 
    186     ASSERT_EQ(tok::text,         Toks[3].getKind());
    187     ASSERT_EQ(StringRef(" Bbb"), Toks[3].getText());
    188     ASSERT_EQ(tok::text,         Toks[4].getKind());
    189     ASSERT_EQ(StringRef("\\"),   Toks[4].getText());
    190     ASSERT_EQ(tok::text,         Toks[5].getKind());
    191     ASSERT_EQ(StringRef(" "),    Toks[5].getText());
    192     ASSERT_EQ(tok::newline,      Toks[6].getKind());
    193 
    194     ASSERT_EQ(tok::text,         Toks[7].getKind());
    195     ASSERT_EQ(StringRef(" Ccc?" "?/"), Toks[7].getText());
    196     ASSERT_EQ(tok::newline,      Toks[8].getKind());
    197 
    198     ASSERT_EQ(tok::newline,      Toks[9].getKind());
    199   }
    200 }
    201 
    202 // Check that we skip C-style aligned stars correctly.
    203 TEST_F(CommentLexerTest, Basic7) {
    204   const char *Source =
    205     "/* Aaa\n"
    206     " * Bbb\r\n"
    207     "\t* Ccc\n"
    208     "  ! Ddd\n"
    209     "  * Eee\n"
    210     "  ** Fff\n"
    211     " */";
    212   std::vector<Token> Toks;
    213 
    214   lexString(Source, Toks);
    215 
    216   ASSERT_EQ(15U, Toks.size());
    217 
    218   ASSERT_EQ(tok::text,         Toks[0].getKind());
    219   ASSERT_EQ(StringRef(" Aaa"), Toks[0].getText());
    220   ASSERT_EQ(tok::newline,      Toks[1].getKind());
    221 
    222   ASSERT_EQ(tok::text,         Toks[2].getKind());
    223   ASSERT_EQ(StringRef(" Bbb"), Toks[2].getText());
    224   ASSERT_EQ(tok::newline,      Toks[3].getKind());
    225 
    226   ASSERT_EQ(tok::text,         Toks[4].getKind());
    227   ASSERT_EQ(StringRef(" Ccc"), Toks[4].getText());
    228   ASSERT_EQ(tok::newline,      Toks[5].getKind());
    229 
    230   ASSERT_EQ(tok::text,            Toks[6].getKind());
    231   ASSERT_EQ(StringRef("  ! Ddd"), Toks[6].getText());
    232   ASSERT_EQ(tok::newline,         Toks[7].getKind());
    233 
    234   ASSERT_EQ(tok::text,         Toks[8].getKind());
    235   ASSERT_EQ(StringRef(" Eee"), Toks[8].getText());
    236   ASSERT_EQ(tok::newline,      Toks[9].getKind());
    237 
    238   ASSERT_EQ(tok::text,          Toks[10].getKind());
    239   ASSERT_EQ(StringRef("* Fff"), Toks[10].getText());
    240   ASSERT_EQ(tok::newline,       Toks[11].getKind());
    241 
    242   ASSERT_EQ(tok::text,         Toks[12].getKind());
    243   ASSERT_EQ(StringRef(" "),    Toks[12].getText());
    244 
    245   ASSERT_EQ(tok::newline,      Toks[13].getKind());
    246   ASSERT_EQ(tok::newline,      Toks[14].getKind());
    247 }
    248 
    249 // A command marker followed by comment end.
    250 TEST_F(CommentLexerTest, DoxygenCommand1) {
    251   const char *Sources[] = { "//@", "///@", "//!@" };
    252   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
    253     std::vector<Token> Toks;
    254 
    255     lexString(Sources[i], Toks);
    256 
    257     ASSERT_EQ(2U, Toks.size());
    258 
    259     ASSERT_EQ(tok::text,          Toks[0].getKind());
    260     ASSERT_EQ(StringRef("@"),     Toks[0].getText());
    261 
    262     ASSERT_EQ(tok::newline,       Toks[1].getKind());
    263   }
    264 }
    265 
    266 // A command marker followed by comment end.
    267 TEST_F(CommentLexerTest, DoxygenCommand2) {
    268   const char *Sources[] = { "/*@*/", "/**@*/", "/*!@*/"};
    269   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
    270     std::vector<Token> Toks;
    271 
    272     lexString(Sources[i], Toks);
    273 
    274     ASSERT_EQ(3U, Toks.size());
    275 
    276     ASSERT_EQ(tok::text,          Toks[0].getKind());
    277     ASSERT_EQ(StringRef("@"),     Toks[0].getText());
    278 
    279     ASSERT_EQ(tok::newline,       Toks[1].getKind());
    280     ASSERT_EQ(tok::newline,       Toks[2].getKind());
    281   }
    282 }
    283 
    284 // A command marker followed by comment end.
    285 TEST_F(CommentLexerTest, DoxygenCommand3) {
    286   const char *Sources[] = { "/*\\*/", "/**\\*/" };
    287   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
    288     std::vector<Token> Toks;
    289 
    290     lexString(Sources[i], Toks);
    291 
    292     ASSERT_EQ(3U, Toks.size());
    293 
    294     ASSERT_EQ(tok::text,           Toks[0].getKind());
    295     ASSERT_EQ(StringRef("\\"),     Toks[0].getText());
    296 
    297     ASSERT_EQ(tok::newline,        Toks[1].getKind());
    298     ASSERT_EQ(tok::newline,        Toks[2].getKind());
    299   }
    300 }
    301 
    302 // Doxygen escape sequences.
    303 TEST_F(CommentLexerTest, DoxygenCommand4) {
    304   const char *Sources[] = {
    305     "/// \\\\ \\@ \\& \\$ \\# \\< \\> \\% \\\" \\. \\::",
    306     "/// @\\ @@ @& @$ @# @< @> @% @\" @. @::"
    307   };
    308   const char *Text[] = {
    309     " ",
    310     "\\", " ", "@", " ", "&", " ", "$",  " ", "#", " ",
    311     "<",  " ", ">", " ", "%", " ", "\"", " ", ".", " ",
    312     "::", ""
    313   };
    314 
    315   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
    316     std::vector<Token> Toks;
    317 
    318     lexString(Sources[i], Toks);
    319 
    320     ASSERT_EQ(array_lengthof(Text), Toks.size());
    321 
    322     for (size_t j = 0, e = Toks.size(); j != e; j++) {
    323       if(Toks[j].is(tok::text))
    324         ASSERT_EQ(StringRef(Text[j]), Toks[j].getText())
    325           << "index " << i;
    326     }
    327   }
    328 }
    329 
    330 // A command marker followed by a non-letter that is not a part of an escape
    331 // sequence.
    332 TEST_F(CommentLexerTest, DoxygenCommand5) {
    333   const char *Source = "/// \\^ \\0";
    334   std::vector<Token> Toks;
    335 
    336   lexString(Source, Toks);
    337 
    338   ASSERT_EQ(6U, Toks.size());
    339 
    340   ASSERT_EQ(tok::text,       Toks[0].getKind());
    341   ASSERT_EQ(StringRef(" "),  Toks[0].getText());
    342 
    343   ASSERT_EQ(tok::text,       Toks[1].getKind());
    344   ASSERT_EQ(StringRef("\\"), Toks[1].getText());
    345 
    346   ASSERT_EQ(tok::text,       Toks[2].getKind());
    347   ASSERT_EQ(StringRef("^ "), Toks[2].getText());
    348 
    349   ASSERT_EQ(tok::text,       Toks[3].getKind());
    350   ASSERT_EQ(StringRef("\\"), Toks[3].getText());
    351 
    352   ASSERT_EQ(tok::text,       Toks[4].getKind());
    353   ASSERT_EQ(StringRef("0"),  Toks[4].getText());
    354 
    355   ASSERT_EQ(tok::newline,    Toks[5].getKind());
    356 }
    357 
    358 TEST_F(CommentLexerTest, DoxygenCommand6) {
    359   const char *Source = "/// \\brief Aaa.";
    360   std::vector<Token> Toks;
    361 
    362   lexString(Source, Toks);
    363 
    364   ASSERT_EQ(4U, Toks.size());
    365 
    366   ASSERT_EQ(tok::text,          Toks[0].getKind());
    367   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
    368 
    369   ASSERT_EQ(tok::backslash_command, Toks[1].getKind());
    370   ASSERT_EQ(StringRef("brief"), getCommandName(Toks[1]));
    371 
    372   ASSERT_EQ(tok::text,          Toks[2].getKind());
    373   ASSERT_EQ(StringRef(" Aaa."), Toks[2].getText());
    374 
    375   ASSERT_EQ(tok::newline,       Toks[3].getKind());
    376 }
    377 
    378 TEST_F(CommentLexerTest, DoxygenCommand7) {
    379   const char *Source = "/// \\em\\em \\em\t\\em\n";
    380   std::vector<Token> Toks;
    381 
    382   lexString(Source, Toks);
    383 
    384   ASSERT_EQ(8U, Toks.size());
    385 
    386   ASSERT_EQ(tok::text,       Toks[0].getKind());
    387   ASSERT_EQ(StringRef(" "),  Toks[0].getText());
    388 
    389   ASSERT_EQ(tok::backslash_command, Toks[1].getKind());
    390   ASSERT_EQ(StringRef("em"), getCommandName(Toks[1]));
    391 
    392   ASSERT_EQ(tok::backslash_command, Toks[2].getKind());
    393   ASSERT_EQ(StringRef("em"), getCommandName(Toks[2]));
    394 
    395   ASSERT_EQ(tok::text,       Toks[3].getKind());
    396   ASSERT_EQ(StringRef(" "),  Toks[3].getText());
    397 
    398   ASSERT_EQ(tok::backslash_command, Toks[4].getKind());
    399   ASSERT_EQ(StringRef("em"), getCommandName(Toks[4]));
    400 
    401   ASSERT_EQ(tok::text,       Toks[5].getKind());
    402   ASSERT_EQ(StringRef("\t"), Toks[5].getText());
    403 
    404   ASSERT_EQ(tok::backslash_command, Toks[6].getKind());
    405   ASSERT_EQ(StringRef("em"), getCommandName(Toks[6]));
    406 
    407   ASSERT_EQ(tok::newline,    Toks[7].getKind());
    408 }
    409 
    410 TEST_F(CommentLexerTest, DoxygenCommand8) {
    411   const char *Source = "/// @em@em @em\t@em\n";
    412   std::vector<Token> Toks;
    413 
    414   lexString(Source, Toks);
    415 
    416   ASSERT_EQ(8U, Toks.size());
    417 
    418   ASSERT_EQ(tok::text,       Toks[0].getKind());
    419   ASSERT_EQ(StringRef(" "),  Toks[0].getText());
    420 
    421   ASSERT_EQ(tok::at_command, Toks[1].getKind());
    422   ASSERT_EQ(StringRef("em"), getCommandName(Toks[1]));
    423 
    424   ASSERT_EQ(tok::at_command, Toks[2].getKind());
    425   ASSERT_EQ(StringRef("em"), getCommandName(Toks[2]));
    426 
    427   ASSERT_EQ(tok::text,       Toks[3].getKind());
    428   ASSERT_EQ(StringRef(" "),  Toks[3].getText());
    429 
    430   ASSERT_EQ(tok::at_command, Toks[4].getKind());
    431   ASSERT_EQ(StringRef("em"), getCommandName(Toks[4]));
    432 
    433   ASSERT_EQ(tok::text,       Toks[5].getKind());
    434   ASSERT_EQ(StringRef("\t"), Toks[5].getText());
    435 
    436   ASSERT_EQ(tok::at_command, Toks[6].getKind());
    437   ASSERT_EQ(StringRef("em"), getCommandName(Toks[6]));
    438 
    439   ASSERT_EQ(tok::newline,    Toks[7].getKind());
    440 }
    441 
    442 TEST_F(CommentLexerTest, DoxygenCommand9) {
    443   const char *Source = "/// \\aaa\\bbb \\ccc\t\\ddd\n";
    444   std::vector<Token> Toks;
    445 
    446   lexString(Source, Toks);
    447 
    448   ASSERT_EQ(8U, Toks.size());
    449 
    450   ASSERT_EQ(tok::text,        Toks[0].getKind());
    451   ASSERT_EQ(StringRef(" "),   Toks[0].getText());
    452 
    453   ASSERT_EQ(tok::unknown_command, Toks[1].getKind());
    454   ASSERT_EQ(StringRef("aaa"), Toks[1].getUnknownCommandName());
    455 
    456   ASSERT_EQ(tok::unknown_command, Toks[2].getKind());
    457   ASSERT_EQ(StringRef("bbb"), Toks[2].getUnknownCommandName());
    458 
    459   ASSERT_EQ(tok::text,        Toks[3].getKind());
    460   ASSERT_EQ(StringRef(" "),   Toks[3].getText());
    461 
    462   ASSERT_EQ(tok::unknown_command, Toks[4].getKind());
    463   ASSERT_EQ(StringRef("ccc"), Toks[4].getUnknownCommandName());
    464 
    465   ASSERT_EQ(tok::text,        Toks[5].getKind());
    466   ASSERT_EQ(StringRef("\t"),  Toks[5].getText());
    467 
    468   ASSERT_EQ(tok::unknown_command, Toks[6].getKind());
    469   ASSERT_EQ(StringRef("ddd"), Toks[6].getUnknownCommandName());
    470 
    471   ASSERT_EQ(tok::newline,     Toks[7].getKind());
    472 }
    473 
    474 TEST_F(CommentLexerTest, DoxygenCommand10) {
    475   const char *Source = "// \\c\n";
    476   std::vector<Token> Toks;
    477 
    478   lexString(Source, Toks);
    479 
    480   ASSERT_EQ(3U, Toks.size());
    481 
    482   ASSERT_EQ(tok::text,      Toks[0].getKind());
    483   ASSERT_EQ(StringRef(" "), Toks[0].getText());
    484 
    485   ASSERT_EQ(tok::backslash_command, Toks[1].getKind());
    486   ASSERT_EQ(StringRef("c"), getCommandName(Toks[1]));
    487 
    488   ASSERT_EQ(tok::newline,   Toks[2].getKind());
    489 }
    490 
    491 TEST_F(CommentLexerTest, RegisterCustomBlockCommand) {
    492   const char *Source =
    493     "/// \\NewBlockCommand Aaa.\n"
    494     "/// @NewBlockCommand Aaa.\n";
    495 
    496   Traits.registerBlockCommand(StringRef("NewBlockCommand"));
    497 
    498   std::vector<Token> Toks;
    499 
    500   lexString(Source, Toks);
    501 
    502   ASSERT_EQ(8U, Toks.size());
    503 
    504   ASSERT_EQ(tok::text,          Toks[0].getKind());
    505   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
    506 
    507   ASSERT_EQ(tok::backslash_command, Toks[1].getKind());
    508   ASSERT_EQ(StringRef("NewBlockCommand"), getCommandName(Toks[1]));
    509 
    510   ASSERT_EQ(tok::text,          Toks[2].getKind());
    511   ASSERT_EQ(StringRef(" Aaa."), Toks[2].getText());
    512 
    513   ASSERT_EQ(tok::newline,       Toks[3].getKind());
    514 
    515   ASSERT_EQ(tok::text,          Toks[4].getKind());
    516   ASSERT_EQ(StringRef(" "),     Toks[4].getText());
    517 
    518   ASSERT_EQ(tok::at_command,    Toks[5].getKind());
    519   ASSERT_EQ(StringRef("NewBlockCommand"), getCommandName(Toks[5]));
    520 
    521   ASSERT_EQ(tok::text,          Toks[6].getKind());
    522   ASSERT_EQ(StringRef(" Aaa."), Toks[6].getText());
    523 
    524   ASSERT_EQ(tok::newline,       Toks[7].getKind());
    525 }
    526 
    527 TEST_F(CommentLexerTest, RegisterMultipleBlockCommands) {
    528   const char *Source =
    529     "/// \\Foo\n"
    530     "/// \\Bar Baz\n"
    531     "/// \\Blech quux=corge\n";
    532 
    533   Traits.registerBlockCommand(StringRef("Foo"));
    534   Traits.registerBlockCommand(StringRef("Bar"));
    535   Traits.registerBlockCommand(StringRef("Blech"));
    536 
    537   std::vector<Token> Toks;
    538 
    539   lexString(Source, Toks);
    540 
    541   ASSERT_EQ(11U, Toks.size());
    542 
    543   ASSERT_EQ(tok::text,      Toks[0].getKind());
    544   ASSERT_EQ(StringRef(" "), Toks[0].getText());
    545 
    546   ASSERT_EQ(tok::backslash_command, Toks[1].getKind());
    547   ASSERT_EQ(StringRef("Foo"), getCommandName(Toks[1]));
    548 
    549   ASSERT_EQ(tok::newline,     Toks[2].getKind());
    550 
    551   ASSERT_EQ(tok::text,      Toks[3].getKind());
    552   ASSERT_EQ(StringRef(" "), Toks[3].getText());
    553 
    554   ASSERT_EQ(tok::backslash_command, Toks[4].getKind());
    555   ASSERT_EQ(StringRef("Bar"), getCommandName(Toks[4]));
    556 
    557   ASSERT_EQ(tok::text,         Toks[5].getKind());
    558   ASSERT_EQ(StringRef(" Baz"), Toks[5].getText());
    559 
    560   ASSERT_EQ(tok::newline,     Toks[6].getKind());
    561 
    562   ASSERT_EQ(tok::text,      Toks[7].getKind());
    563   ASSERT_EQ(StringRef(" "), Toks[7].getText());
    564 
    565   ASSERT_EQ(tok::backslash_command, Toks[8].getKind());
    566   ASSERT_EQ(StringRef("Blech"), getCommandName(Toks[8]));
    567 
    568   ASSERT_EQ(tok::text,                Toks[9].getKind());
    569   ASSERT_EQ(StringRef(" quux=corge"), Toks[9].getText());
    570 
    571   ASSERT_EQ(tok::newline,     Toks[10].getKind());
    572 }
    573 
    574 // Empty verbatim block.
    575 TEST_F(CommentLexerTest, VerbatimBlock1) {
    576   const char *Sources[] = {
    577     "/// \\verbatim\\endverbatim\n//",
    578     "/** \\verbatim\\endverbatim*/"
    579   };
    580 
    581   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
    582     std::vector<Token> Toks;
    583 
    584     lexString(Sources[i], Toks);
    585 
    586     ASSERT_EQ(5U, Toks.size());
    587 
    588     ASSERT_EQ(tok::text,                 Toks[0].getKind());
    589     ASSERT_EQ(StringRef(" "),            Toks[0].getText());
    590 
    591     ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
    592     ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
    593 
    594     ASSERT_EQ(tok::verbatim_block_end,   Toks[2].getKind());
    595     ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[2]));
    596 
    597     ASSERT_EQ(tok::newline,              Toks[3].getKind());
    598     ASSERT_EQ(tok::newline,              Toks[4].getKind());
    599   }
    600 }
    601 
    602 // Empty verbatim block without an end command.
    603 TEST_F(CommentLexerTest, VerbatimBlock2) {
    604   const char *Source = "/// \\verbatim";
    605 
    606   std::vector<Token> Toks;
    607 
    608   lexString(Source, Toks);
    609 
    610   ASSERT_EQ(3U, Toks.size());
    611 
    612   ASSERT_EQ(tok::text,                 Toks[0].getKind());
    613   ASSERT_EQ(StringRef(" "),            Toks[0].getText());
    614 
    615   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
    616   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
    617 
    618   ASSERT_EQ(tok::newline,              Toks[2].getKind());
    619 }
    620 
    621 // Empty verbatim block without an end command.
    622 TEST_F(CommentLexerTest, VerbatimBlock3) {
    623   const char *Source = "/** \\verbatim*/";
    624 
    625   std::vector<Token> Toks;
    626 
    627   lexString(Source, Toks);
    628 
    629   ASSERT_EQ(4U, Toks.size());
    630 
    631   ASSERT_EQ(tok::text,                 Toks[0].getKind());
    632   ASSERT_EQ(StringRef(" "),            Toks[0].getText());
    633 
    634   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
    635   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
    636 
    637   ASSERT_EQ(tok::newline,              Toks[2].getKind());
    638   ASSERT_EQ(tok::newline,              Toks[3].getKind());
    639 }
    640 
    641 // Single-line verbatim block.
    642 TEST_F(CommentLexerTest, VerbatimBlock4) {
    643   const char *Sources[] = {
    644     "/// Meow \\verbatim aaa \\endverbatim\n//",
    645     "/** Meow \\verbatim aaa \\endverbatim*/"
    646   };
    647 
    648   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
    649     std::vector<Token> Toks;
    650 
    651     lexString(Sources[i], Toks);
    652 
    653     ASSERT_EQ(6U, Toks.size());
    654 
    655     ASSERT_EQ(tok::text,                 Toks[0].getKind());
    656     ASSERT_EQ(StringRef(" Meow "),       Toks[0].getText());
    657 
    658     ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
    659     ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
    660 
    661     ASSERT_EQ(tok::verbatim_block_line,  Toks[2].getKind());
    662     ASSERT_EQ(StringRef(" aaa "),        Toks[2].getVerbatimBlockText());
    663 
    664     ASSERT_EQ(tok::verbatim_block_end,   Toks[3].getKind());
    665     ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[3]));
    666 
    667     ASSERT_EQ(tok::newline,              Toks[4].getKind());
    668     ASSERT_EQ(tok::newline,              Toks[5].getKind());
    669   }
    670 }
    671 
    672 // Single-line verbatim block without an end command.
    673 TEST_F(CommentLexerTest, VerbatimBlock5) {
    674   const char *Sources[] = {
    675     "/// Meow \\verbatim aaa \n//",
    676     "/** Meow \\verbatim aaa */"
    677   };
    678 
    679   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
    680     std::vector<Token> Toks;
    681 
    682     lexString(Sources[i], Toks);
    683 
    684     ASSERT_EQ(5U, Toks.size());
    685 
    686     ASSERT_EQ(tok::text,                 Toks[0].getKind());
    687     ASSERT_EQ(StringRef(" Meow "),       Toks[0].getText());
    688 
    689     ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
    690     ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
    691 
    692     ASSERT_EQ(tok::verbatim_block_line,  Toks[2].getKind());
    693     ASSERT_EQ(StringRef(" aaa "),        Toks[2].getVerbatimBlockText());
    694 
    695     ASSERT_EQ(tok::newline,              Toks[3].getKind());
    696     ASSERT_EQ(tok::newline,              Toks[4].getKind());
    697   }
    698 }
    699 
    700 TEST_F(CommentLexerTest, VerbatimBlock6) {
    701   const char *Source =
    702     "// \\verbatim\n"
    703     "// Aaa\n"
    704     "//\n"
    705     "// Bbb\n"
    706     "// \\endverbatim\n";
    707 
    708   std::vector<Token> Toks;
    709 
    710   lexString(Source, Toks);
    711 
    712   ASSERT_EQ(10U, Toks.size());
    713 
    714   ASSERT_EQ(tok::text,                 Toks[0].getKind());
    715   ASSERT_EQ(StringRef(" "),            Toks[0].getText());
    716 
    717   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
    718   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
    719 
    720   ASSERT_EQ(tok::newline,              Toks[2].getKind());
    721 
    722   ASSERT_EQ(tok::verbatim_block_line,  Toks[3].getKind());
    723   ASSERT_EQ(StringRef(" Aaa"),         Toks[3].getVerbatimBlockText());
    724 
    725   ASSERT_EQ(tok::newline,              Toks[4].getKind());
    726 
    727   ASSERT_EQ(tok::newline,              Toks[5].getKind());
    728 
    729   ASSERT_EQ(tok::verbatim_block_line,  Toks[6].getKind());
    730   ASSERT_EQ(StringRef(" Bbb"),         Toks[6].getVerbatimBlockText());
    731 
    732   ASSERT_EQ(tok::newline,              Toks[7].getKind());
    733 
    734   ASSERT_EQ(tok::verbatim_block_end,   Toks[8].getKind());
    735   ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[8]));
    736 
    737   ASSERT_EQ(tok::newline,              Toks[9].getKind());
    738 }
    739 
    740 TEST_F(CommentLexerTest, VerbatimBlock7) {
    741   const char *Source =
    742     "/* \\verbatim\n"
    743     " * Aaa\n"
    744     " *\n"
    745     " * Bbb\n"
    746     " * \\endverbatim\n"
    747     " */";
    748 
    749   std::vector<Token> Toks;
    750 
    751   lexString(Source, Toks);
    752 
    753   ASSERT_EQ(10U, Toks.size());
    754 
    755   ASSERT_EQ(tok::text,                 Toks[0].getKind());
    756   ASSERT_EQ(StringRef(" "),            Toks[0].getText());
    757 
    758   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
    759   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
    760 
    761   ASSERT_EQ(tok::verbatim_block_line,  Toks[2].getKind());
    762   ASSERT_EQ(StringRef(" Aaa"),         Toks[2].getVerbatimBlockText());
    763 
    764   ASSERT_EQ(tok::verbatim_block_line,  Toks[3].getKind());
    765   ASSERT_EQ(StringRef(""),             Toks[3].getVerbatimBlockText());
    766 
    767   ASSERT_EQ(tok::verbatim_block_line,  Toks[4].getKind());
    768   ASSERT_EQ(StringRef(" Bbb"),         Toks[4].getVerbatimBlockText());
    769 
    770   ASSERT_EQ(tok::verbatim_block_end,   Toks[5].getKind());
    771   ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[5]));
    772 
    773   ASSERT_EQ(tok::newline,              Toks[6].getKind());
    774 
    775   ASSERT_EQ(tok::text,                 Toks[7].getKind());
    776   ASSERT_EQ(StringRef(" "),            Toks[7].getText());
    777 
    778   ASSERT_EQ(tok::newline,              Toks[8].getKind());
    779   ASSERT_EQ(tok::newline,              Toks[9].getKind());
    780 }
    781 
    782 // Complex test for verbatim blocks.
    783 TEST_F(CommentLexerTest, VerbatimBlock8) {
    784   const char *Source =
    785     "/* Meow \\verbatim aaa\\$\\@\n"
    786     "bbb \\endverbati\r"
    787     "ccc\r\n"
    788     "ddd \\endverbatim Blah \\verbatim eee\n"
    789     "\\endverbatim BlahBlah*/";
    790   std::vector<Token> Toks;
    791 
    792   lexString(Source, Toks);
    793 
    794   ASSERT_EQ(14U, Toks.size());
    795 
    796   ASSERT_EQ(tok::text,                 Toks[0].getKind());
    797   ASSERT_EQ(StringRef(" Meow "),       Toks[0].getText());
    798 
    799   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
    800   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[1]));
    801 
    802   ASSERT_EQ(tok::verbatim_block_line,  Toks[2].getKind());
    803   ASSERT_EQ(StringRef(" aaa\\$\\@"),   Toks[2].getVerbatimBlockText());
    804 
    805   ASSERT_EQ(tok::verbatim_block_line,  Toks[3].getKind());
    806   ASSERT_EQ(StringRef("bbb \\endverbati"), Toks[3].getVerbatimBlockText());
    807 
    808   ASSERT_EQ(tok::verbatim_block_line,  Toks[4].getKind());
    809   ASSERT_EQ(StringRef("ccc"),          Toks[4].getVerbatimBlockText());
    810 
    811   ASSERT_EQ(tok::verbatim_block_line,  Toks[5].getKind());
    812   ASSERT_EQ(StringRef("ddd "),         Toks[5].getVerbatimBlockText());
    813 
    814   ASSERT_EQ(tok::verbatim_block_end,   Toks[6].getKind());
    815   ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[6]));
    816 
    817   ASSERT_EQ(tok::text,                 Toks[7].getKind());
    818   ASSERT_EQ(StringRef(" Blah "),       Toks[7].getText());
    819 
    820   ASSERT_EQ(tok::verbatim_block_begin, Toks[8].getKind());
    821   ASSERT_EQ(StringRef("verbatim"),     getVerbatimBlockName(Toks[8]));
    822 
    823   ASSERT_EQ(tok::verbatim_block_line,  Toks[9].getKind());
    824   ASSERT_EQ(StringRef(" eee"),         Toks[9].getVerbatimBlockText());
    825 
    826   ASSERT_EQ(tok::verbatim_block_end,   Toks[10].getKind());
    827   ASSERT_EQ(StringRef("endverbatim"),  getVerbatimBlockName(Toks[10]));
    828 
    829   ASSERT_EQ(tok::text,                 Toks[11].getKind());
    830   ASSERT_EQ(StringRef(" BlahBlah"),    Toks[11].getText());
    831 
    832   ASSERT_EQ(tok::newline,              Toks[12].getKind());
    833   ASSERT_EQ(tok::newline,              Toks[13].getKind());
    834 }
    835 
    836 // LaTeX verbatim blocks.
    837 TEST_F(CommentLexerTest, VerbatimBlock9) {
    838   const char *Source =
    839     "/// \\f$ Aaa \\f$ \\f[ Bbb \\f] \\f{ Ccc \\f}";
    840   std::vector<Token> Toks;
    841 
    842   lexString(Source, Toks);
    843 
    844   ASSERT_EQ(13U, Toks.size());
    845 
    846   ASSERT_EQ(tok::text,                 Toks[0].getKind());
    847   ASSERT_EQ(StringRef(" "),            Toks[0].getText());
    848 
    849   ASSERT_EQ(tok::verbatim_block_begin, Toks[1].getKind());
    850   ASSERT_EQ(StringRef("f$"),           getVerbatimBlockName(Toks[1]));
    851 
    852   ASSERT_EQ(tok::verbatim_block_line,  Toks[2].getKind());
    853   ASSERT_EQ(StringRef(" Aaa "),        Toks[2].getVerbatimBlockText());
    854 
    855   ASSERT_EQ(tok::verbatim_block_end,   Toks[3].getKind());
    856   ASSERT_EQ(StringRef("f$"),           getVerbatimBlockName(Toks[3]));
    857 
    858   ASSERT_EQ(tok::text,                 Toks[4].getKind());
    859   ASSERT_EQ(StringRef(" "),            Toks[4].getText());
    860 
    861   ASSERT_EQ(tok::verbatim_block_begin, Toks[5].getKind());
    862   ASSERT_EQ(StringRef("f["),           getVerbatimBlockName(Toks[5]));
    863 
    864   ASSERT_EQ(tok::verbatim_block_line,  Toks[6].getKind());
    865   ASSERT_EQ(StringRef(" Bbb "),        Toks[6].getVerbatimBlockText());
    866 
    867   ASSERT_EQ(tok::verbatim_block_end,   Toks[7].getKind());
    868   ASSERT_EQ(StringRef("f]"),           getVerbatimBlockName(Toks[7]));
    869 
    870   ASSERT_EQ(tok::text,                 Toks[8].getKind());
    871   ASSERT_EQ(StringRef(" "),            Toks[8].getText());
    872 
    873   ASSERT_EQ(tok::verbatim_block_begin, Toks[9].getKind());
    874   ASSERT_EQ(StringRef("f{"),           getVerbatimBlockName(Toks[9]));
    875 
    876   ASSERT_EQ(tok::verbatim_block_line,  Toks[10].getKind());
    877   ASSERT_EQ(StringRef(" Ccc "),        Toks[10].getVerbatimBlockText());
    878 
    879   ASSERT_EQ(tok::verbatim_block_end,   Toks[11].getKind());
    880   ASSERT_EQ(StringRef("f}"),           getVerbatimBlockName(Toks[11]));
    881 
    882   ASSERT_EQ(tok::newline,              Toks[12].getKind());
    883 }
    884 
    885 // Empty verbatim line.
    886 TEST_F(CommentLexerTest, VerbatimLine1) {
    887   const char *Sources[] = {
    888     "/// \\fn\n//",
    889     "/** \\fn*/"
    890   };
    891 
    892   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
    893     std::vector<Token> Toks;
    894 
    895     lexString(Sources[i], Toks);
    896 
    897     ASSERT_EQ(4U, Toks.size());
    898 
    899     ASSERT_EQ(tok::text,               Toks[0].getKind());
    900     ASSERT_EQ(StringRef(" "),          Toks[0].getText());
    901 
    902     ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());
    903     ASSERT_EQ(StringRef("fn"),         getVerbatimLineName(Toks[1]));
    904 
    905     ASSERT_EQ(tok::newline,            Toks[2].getKind());
    906     ASSERT_EQ(tok::newline,            Toks[3].getKind());
    907   }
    908 }
    909 
    910 // Verbatim line with Doxygen escape sequences, which should not be expanded.
    911 TEST_F(CommentLexerTest, VerbatimLine2) {
    912   const char *Sources[] = {
    913     "/// \\fn void *foo(const char *zzz = \"\\$\");\n//",
    914     "/** \\fn void *foo(const char *zzz = \"\\$\");*/"
    915   };
    916 
    917   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
    918     std::vector<Token> Toks;
    919 
    920     lexString(Sources[i], Toks);
    921 
    922     ASSERT_EQ(5U, Toks.size());
    923 
    924     ASSERT_EQ(tok::text,               Toks[0].getKind());
    925     ASSERT_EQ(StringRef(" "),          Toks[0].getText());
    926 
    927     ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());
    928     ASSERT_EQ(StringRef("fn"),         getVerbatimLineName(Toks[1]));
    929 
    930     ASSERT_EQ(tok::verbatim_line_text, Toks[2].getKind());
    931     ASSERT_EQ(StringRef(" void *foo(const char *zzz = \"\\$\");"),
    932                                        Toks[2].getVerbatimLineText());
    933 
    934     ASSERT_EQ(tok::newline,            Toks[3].getKind());
    935     ASSERT_EQ(tok::newline,            Toks[4].getKind());
    936   }
    937 }
    938 
    939 // Verbatim line should not eat anything from next source line.
    940 TEST_F(CommentLexerTest, VerbatimLine3) {
    941   const char *Source =
    942     "/** \\fn void *foo(const char *zzz = \"\\$\");\n"
    943     " * Meow\n"
    944     " */";
    945 
    946   std::vector<Token> Toks;
    947 
    948   lexString(Source, Toks);
    949 
    950   ASSERT_EQ(9U, Toks.size());
    951 
    952   ASSERT_EQ(tok::text,               Toks[0].getKind());
    953   ASSERT_EQ(StringRef(" "),          Toks[0].getText());
    954 
    955   ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());
    956   ASSERT_EQ(StringRef("fn"),         getVerbatimLineName(Toks[1]));
    957 
    958   ASSERT_EQ(tok::verbatim_line_text, Toks[2].getKind());
    959   ASSERT_EQ(StringRef(" void *foo(const char *zzz = \"\\$\");"),
    960                                      Toks[2].getVerbatimLineText());
    961   ASSERT_EQ(tok::newline,            Toks[3].getKind());
    962 
    963   ASSERT_EQ(tok::text,               Toks[4].getKind());
    964   ASSERT_EQ(StringRef(" Meow"),      Toks[4].getText());
    965   ASSERT_EQ(tok::newline,            Toks[5].getKind());
    966 
    967   ASSERT_EQ(tok::text,               Toks[6].getKind());
    968   ASSERT_EQ(StringRef(" "),          Toks[6].getText());
    969 
    970   ASSERT_EQ(tok::newline,            Toks[7].getKind());
    971   ASSERT_EQ(tok::newline,            Toks[8].getKind());
    972 }
    973 
    974 TEST_F(CommentLexerTest, HTML1) {
    975   const char *Source =
    976     "// <";
    977 
    978   std::vector<Token> Toks;
    979 
    980   lexString(Source, Toks);
    981 
    982   ASSERT_EQ(3U, Toks.size());
    983 
    984   ASSERT_EQ(tok::text,      Toks[0].getKind());
    985   ASSERT_EQ(StringRef(" "), Toks[0].getText());
    986 
    987   ASSERT_EQ(tok::text,      Toks[1].getKind());
    988   ASSERT_EQ(StringRef("<"), Toks[1].getText());
    989 
    990   ASSERT_EQ(tok::newline,   Toks[2].getKind());
    991 }
    992 
    993 TEST_F(CommentLexerTest, HTML2) {
    994   const char *Source =
    995     "// a<2";
    996 
    997   std::vector<Token> Toks;
    998 
    999   lexString(Source, Toks);
   1000 
   1001   ASSERT_EQ(4U, Toks.size());
   1002 
   1003   ASSERT_EQ(tok::text,       Toks[0].getKind());
   1004   ASSERT_EQ(StringRef(" a"), Toks[0].getText());
   1005 
   1006   ASSERT_EQ(tok::text,       Toks[1].getKind());
   1007   ASSERT_EQ(StringRef("<"),  Toks[1].getText());
   1008 
   1009   ASSERT_EQ(tok::text,       Toks[2].getKind());
   1010   ASSERT_EQ(StringRef("2"),  Toks[2].getText());
   1011 
   1012   ASSERT_EQ(tok::newline,    Toks[3].getKind());
   1013 }
   1014 
   1015 TEST_F(CommentLexerTest, HTML3) {
   1016   const char *Source =
   1017     "// < img";
   1018 
   1019   std::vector<Token> Toks;
   1020 
   1021   lexString(Source, Toks);
   1022 
   1023   ASSERT_EQ(4U, Toks.size());
   1024 
   1025   ASSERT_EQ(tok::text,         Toks[0].getKind());
   1026   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
   1027 
   1028   ASSERT_EQ(tok::text,         Toks[1].getKind());
   1029   ASSERT_EQ(StringRef("<"),    Toks[1].getText());
   1030 
   1031   ASSERT_EQ(tok::text,         Toks[2].getKind());
   1032   ASSERT_EQ(StringRef(" img"), Toks[2].getText());
   1033 
   1034   ASSERT_EQ(tok::newline,      Toks[3].getKind());
   1035 }
   1036 
   1037 TEST_F(CommentLexerTest, HTML4) {
   1038   const char *Sources[] = {
   1039     "// <img",
   1040     "// <img "
   1041   };
   1042 
   1043   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
   1044     std::vector<Token> Toks;
   1045 
   1046     lexString(Sources[i], Toks);
   1047 
   1048     ASSERT_EQ(3U, Toks.size());
   1049 
   1050     ASSERT_EQ(tok::text,           Toks[0].getKind());
   1051     ASSERT_EQ(StringRef(" "),      Toks[0].getText());
   1052 
   1053     ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
   1054     ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
   1055 
   1056     ASSERT_EQ(tok::newline,        Toks[2].getKind());
   1057   }
   1058 }
   1059 
   1060 TEST_F(CommentLexerTest, HTML5) {
   1061   const char *Source =
   1062     "// <img 42";
   1063 
   1064   std::vector<Token> Toks;
   1065 
   1066   lexString(Source, Toks);
   1067 
   1068   ASSERT_EQ(4U, Toks.size());
   1069 
   1070   ASSERT_EQ(tok::text,           Toks[0].getKind());
   1071   ASSERT_EQ(StringRef(" "),      Toks[0].getText());
   1072 
   1073   ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
   1074   ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
   1075 
   1076   ASSERT_EQ(tok::text,           Toks[2].getKind());
   1077   ASSERT_EQ(StringRef("42"),     Toks[2].getText());
   1078 
   1079   ASSERT_EQ(tok::newline,        Toks[3].getKind());
   1080 }
   1081 
   1082 TEST_F(CommentLexerTest, HTML6) {
   1083   const char *Source = "// <img> Meow";
   1084 
   1085   std::vector<Token> Toks;
   1086 
   1087   lexString(Source, Toks);
   1088 
   1089   ASSERT_EQ(5U, Toks.size());
   1090 
   1091   ASSERT_EQ(tok::text,           Toks[0].getKind());
   1092   ASSERT_EQ(StringRef(" "),      Toks[0].getText());
   1093 
   1094   ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
   1095   ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
   1096 
   1097   ASSERT_EQ(tok::html_greater,   Toks[2].getKind());
   1098 
   1099   ASSERT_EQ(tok::text,           Toks[3].getKind());
   1100   ASSERT_EQ(StringRef(" Meow"),  Toks[3].getText());
   1101 
   1102   ASSERT_EQ(tok::newline,        Toks[4].getKind());
   1103 }
   1104 
   1105 TEST_F(CommentLexerTest, HTML7) {
   1106   const char *Source = "// <img=";
   1107 
   1108   std::vector<Token> Toks;
   1109 
   1110   lexString(Source, Toks);
   1111 
   1112   ASSERT_EQ(4U, Toks.size());
   1113 
   1114   ASSERT_EQ(tok::text,           Toks[0].getKind());
   1115   ASSERT_EQ(StringRef(" "),      Toks[0].getText());
   1116 
   1117   ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
   1118   ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
   1119 
   1120   ASSERT_EQ(tok::text,           Toks[2].getKind());
   1121   ASSERT_EQ(StringRef("="),      Toks[2].getText());
   1122 
   1123   ASSERT_EQ(tok::newline,        Toks[3].getKind());
   1124 }
   1125 
   1126 TEST_F(CommentLexerTest, HTML8) {
   1127   const char *Source = "// <img src=> Meow";
   1128 
   1129   std::vector<Token> Toks;
   1130 
   1131   lexString(Source, Toks);
   1132 
   1133   ASSERT_EQ(7U, Toks.size());
   1134 
   1135   ASSERT_EQ(tok::text,           Toks[0].getKind());
   1136   ASSERT_EQ(StringRef(" "),      Toks[0].getText());
   1137 
   1138   ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
   1139   ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
   1140 
   1141   ASSERT_EQ(tok::html_ident,     Toks[2].getKind());
   1142   ASSERT_EQ(StringRef("src"),   Toks[2].getHTMLIdent());
   1143 
   1144   ASSERT_EQ(tok::html_equals,    Toks[3].getKind());
   1145 
   1146   ASSERT_EQ(tok::html_greater,   Toks[4].getKind());
   1147 
   1148   ASSERT_EQ(tok::text,           Toks[5].getKind());
   1149   ASSERT_EQ(StringRef(" Meow"),  Toks[5].getText());
   1150 
   1151   ASSERT_EQ(tok::newline,        Toks[6].getKind());
   1152 }
   1153 
   1154 TEST_F(CommentLexerTest, HTML9) {
   1155   const char *Sources[] = {
   1156     "// <img src",
   1157     "// <img src "
   1158   };
   1159 
   1160   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
   1161     std::vector<Token> Toks;
   1162 
   1163     lexString(Sources[i], Toks);
   1164 
   1165     ASSERT_EQ(4U, Toks.size());
   1166 
   1167     ASSERT_EQ(tok::text,           Toks[0].getKind());
   1168     ASSERT_EQ(StringRef(" "),      Toks[0].getText());
   1169 
   1170     ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
   1171     ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
   1172 
   1173     ASSERT_EQ(tok::html_ident,     Toks[2].getKind());
   1174     ASSERT_EQ(StringRef("src"),    Toks[2].getHTMLIdent());
   1175 
   1176     ASSERT_EQ(tok::newline,        Toks[3].getKind());
   1177   }
   1178 }
   1179 
   1180 TEST_F(CommentLexerTest, HTML10) {
   1181   const char *Sources[] = {
   1182     "// <img src=",
   1183     "// <img src ="
   1184   };
   1185 
   1186   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
   1187     std::vector<Token> Toks;
   1188 
   1189     lexString(Sources[i], Toks);
   1190 
   1191     ASSERT_EQ(5U, Toks.size());
   1192 
   1193     ASSERT_EQ(tok::text,           Toks[0].getKind());
   1194     ASSERT_EQ(StringRef(" "),      Toks[0].getText());
   1195 
   1196     ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
   1197     ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
   1198 
   1199     ASSERT_EQ(tok::html_ident,     Toks[2].getKind());
   1200     ASSERT_EQ(StringRef("src"),    Toks[2].getHTMLIdent());
   1201 
   1202     ASSERT_EQ(tok::html_equals,    Toks[3].getKind());
   1203 
   1204     ASSERT_EQ(tok::newline,        Toks[4].getKind());
   1205   }
   1206 }
   1207 
   1208 TEST_F(CommentLexerTest, HTML11) {
   1209   const char *Sources[] = {
   1210     "// <img src=\"",
   1211     "// <img src = \"",
   1212     "// <img src=\'",
   1213     "// <img src = \'"
   1214   };
   1215 
   1216   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
   1217     std::vector<Token> Toks;
   1218 
   1219     lexString(Sources[i], Toks);
   1220 
   1221     ASSERT_EQ(6U, Toks.size());
   1222 
   1223     ASSERT_EQ(tok::text,               Toks[0].getKind());
   1224     ASSERT_EQ(StringRef(" "),          Toks[0].getText());
   1225 
   1226     ASSERT_EQ(tok::html_start_tag,     Toks[1].getKind());
   1227     ASSERT_EQ(StringRef("img"),        Toks[1].getHTMLTagStartName());
   1228 
   1229     ASSERT_EQ(tok::html_ident,         Toks[2].getKind());
   1230     ASSERT_EQ(StringRef("src"),        Toks[2].getHTMLIdent());
   1231 
   1232     ASSERT_EQ(tok::html_equals,        Toks[3].getKind());
   1233 
   1234     ASSERT_EQ(tok::html_quoted_string, Toks[4].getKind());
   1235     ASSERT_EQ(StringRef(""),           Toks[4].getHTMLQuotedString());
   1236 
   1237     ASSERT_EQ(tok::newline,            Toks[5].getKind());
   1238   }
   1239 }
   1240 
   1241 TEST_F(CommentLexerTest, HTML12) {
   1242   const char *Source = "// <img src=@";
   1243 
   1244   std::vector<Token> Toks;
   1245 
   1246   lexString(Source, Toks);
   1247 
   1248   ASSERT_EQ(6U, Toks.size());
   1249 
   1250   ASSERT_EQ(tok::text,           Toks[0].getKind());
   1251   ASSERT_EQ(StringRef(" "),      Toks[0].getText());
   1252 
   1253   ASSERT_EQ(tok::html_start_tag, Toks[1].getKind());
   1254   ASSERT_EQ(StringRef("img"),    Toks[1].getHTMLTagStartName());
   1255 
   1256   ASSERT_EQ(tok::html_ident,     Toks[2].getKind());
   1257   ASSERT_EQ(StringRef("src"),    Toks[2].getHTMLIdent());
   1258 
   1259   ASSERT_EQ(tok::html_equals,    Toks[3].getKind());
   1260 
   1261   ASSERT_EQ(tok::text,           Toks[4].getKind());
   1262   ASSERT_EQ(StringRef("@"),      Toks[4].getText());
   1263 
   1264   ASSERT_EQ(tok::newline,        Toks[5].getKind());
   1265 }
   1266 
   1267 TEST_F(CommentLexerTest, HTML13) {
   1268   const char *Sources[] = {
   1269     "// <img src=\"val\\\"\\'val",
   1270     "// <img src=\"val\\\"\\'val\"",
   1271     "// <img src=\'val\\\"\\'val",
   1272     "// <img src=\'val\\\"\\'val\'"
   1273   };
   1274 
   1275   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
   1276     std::vector<Token> Toks;
   1277 
   1278     lexString(Sources[i], Toks);
   1279 
   1280     ASSERT_EQ(6U, Toks.size());
   1281 
   1282     ASSERT_EQ(tok::text,                  Toks[0].getKind());
   1283     ASSERT_EQ(StringRef(" "),             Toks[0].getText());
   1284 
   1285     ASSERT_EQ(tok::html_start_tag,        Toks[1].getKind());
   1286     ASSERT_EQ(StringRef("img"),           Toks[1].getHTMLTagStartName());
   1287 
   1288     ASSERT_EQ(tok::html_ident,            Toks[2].getKind());
   1289     ASSERT_EQ(StringRef("src"),           Toks[2].getHTMLIdent());
   1290 
   1291     ASSERT_EQ(tok::html_equals,           Toks[3].getKind());
   1292 
   1293     ASSERT_EQ(tok::html_quoted_string,    Toks[4].getKind());
   1294     ASSERT_EQ(StringRef("val\\\"\\'val"), Toks[4].getHTMLQuotedString());
   1295 
   1296     ASSERT_EQ(tok::newline,               Toks[5].getKind());
   1297   }
   1298 }
   1299 
   1300 TEST_F(CommentLexerTest, HTML14) {
   1301   const char *Sources[] = {
   1302     "// <img src=\"val\\\"\\'val\">",
   1303     "// <img src=\'val\\\"\\'val\'>"
   1304   };
   1305 
   1306   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
   1307     std::vector<Token> Toks;
   1308 
   1309     lexString(Sources[i], Toks);
   1310 
   1311     ASSERT_EQ(7U, Toks.size());
   1312 
   1313     ASSERT_EQ(tok::text,                  Toks[0].getKind());
   1314     ASSERT_EQ(StringRef(" "),             Toks[0].getText());
   1315 
   1316     ASSERT_EQ(tok::html_start_tag,        Toks[1].getKind());
   1317     ASSERT_EQ(StringRef("img"),           Toks[1].getHTMLTagStartName());
   1318 
   1319     ASSERT_EQ(tok::html_ident,            Toks[2].getKind());
   1320     ASSERT_EQ(StringRef("src"),           Toks[2].getHTMLIdent());
   1321 
   1322     ASSERT_EQ(tok::html_equals,           Toks[3].getKind());
   1323 
   1324     ASSERT_EQ(tok::html_quoted_string,    Toks[4].getKind());
   1325     ASSERT_EQ(StringRef("val\\\"\\'val"), Toks[4].getHTMLQuotedString());
   1326 
   1327     ASSERT_EQ(tok::html_greater,          Toks[5].getKind());
   1328 
   1329     ASSERT_EQ(tok::newline,               Toks[6].getKind());
   1330   }
   1331 }
   1332 
   1333 TEST_F(CommentLexerTest, HTML15) {
   1334   const char *Sources[] = {
   1335     "// <img/>",
   1336     "// <img />"
   1337   };
   1338 
   1339   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
   1340     std::vector<Token> Toks;
   1341 
   1342     lexString(Sources[i], Toks);
   1343 
   1344     ASSERT_EQ(4U, Toks.size());
   1345 
   1346     ASSERT_EQ(tok::text,               Toks[0].getKind());
   1347     ASSERT_EQ(StringRef(" "),          Toks[0].getText());
   1348 
   1349     ASSERT_EQ(tok::html_start_tag,     Toks[1].getKind());
   1350     ASSERT_EQ(StringRef("img"),        Toks[1].getHTMLTagStartName());
   1351 
   1352     ASSERT_EQ(tok::html_slash_greater, Toks[2].getKind());
   1353 
   1354     ASSERT_EQ(tok::newline,            Toks[3].getKind());
   1355   }
   1356 }
   1357 
   1358 TEST_F(CommentLexerTest, HTML16) {
   1359   const char *Sources[] = {
   1360     "// <img/ Aaa",
   1361     "// <img / Aaa"
   1362   };
   1363 
   1364   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
   1365     std::vector<Token> Toks;
   1366 
   1367     lexString(Sources[i], Toks);
   1368 
   1369     ASSERT_EQ(5U, Toks.size());
   1370 
   1371     ASSERT_EQ(tok::text,               Toks[0].getKind());
   1372     ASSERT_EQ(StringRef(" "),          Toks[0].getText());
   1373 
   1374     ASSERT_EQ(tok::html_start_tag,     Toks[1].getKind());
   1375     ASSERT_EQ(StringRef("img"),        Toks[1].getHTMLTagStartName());
   1376 
   1377     ASSERT_EQ(tok::text,               Toks[2].getKind());
   1378     ASSERT_EQ(StringRef("/"),          Toks[2].getText());
   1379 
   1380     ASSERT_EQ(tok::text,               Toks[3].getKind());
   1381     ASSERT_EQ(StringRef(" Aaa"),       Toks[3].getText());
   1382 
   1383     ASSERT_EQ(tok::newline,            Toks[4].getKind());
   1384   }
   1385 }
   1386 
   1387 TEST_F(CommentLexerTest, HTML17) {
   1388   const char *Source = "// </";
   1389 
   1390   std::vector<Token> Toks;
   1391 
   1392   lexString(Source, Toks);
   1393 
   1394   ASSERT_EQ(3U, Toks.size());
   1395 
   1396   ASSERT_EQ(tok::text,       Toks[0].getKind());
   1397   ASSERT_EQ(StringRef(" "),  Toks[0].getText());
   1398 
   1399   ASSERT_EQ(tok::text,       Toks[1].getKind());
   1400   ASSERT_EQ(StringRef("</"), Toks[1].getText());
   1401 
   1402   ASSERT_EQ(tok::newline,    Toks[2].getKind());
   1403 }
   1404 
   1405 TEST_F(CommentLexerTest, HTML18) {
   1406   const char *Source = "// </@";
   1407 
   1408   std::vector<Token> Toks;
   1409 
   1410   lexString(Source, Toks);
   1411 
   1412   ASSERT_EQ(4U, Toks.size());
   1413 
   1414   ASSERT_EQ(tok::text,       Toks[0].getKind());
   1415   ASSERT_EQ(StringRef(" "),  Toks[0].getText());
   1416 
   1417   ASSERT_EQ(tok::text,       Toks[1].getKind());
   1418   ASSERT_EQ(StringRef("</"), Toks[1].getText());
   1419 
   1420   ASSERT_EQ(tok::text,       Toks[2].getKind());
   1421   ASSERT_EQ(StringRef("@"),  Toks[2].getText());
   1422 
   1423   ASSERT_EQ(tok::newline,    Toks[3].getKind());
   1424 }
   1425 
   1426 TEST_F(CommentLexerTest, HTML19) {
   1427   const char *Source = "// </img";
   1428 
   1429   std::vector<Token> Toks;
   1430 
   1431   lexString(Source, Toks);
   1432 
   1433   ASSERT_EQ(3U, Toks.size());
   1434 
   1435   ASSERT_EQ(tok::text,         Toks[0].getKind());
   1436   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
   1437 
   1438   ASSERT_EQ(tok::html_end_tag, Toks[1].getKind());
   1439   ASSERT_EQ(StringRef("img"),  Toks[1].getHTMLTagEndName());
   1440 
   1441   ASSERT_EQ(tok::newline,      Toks[2].getKind());
   1442 }
   1443 
   1444 TEST_F(CommentLexerTest, NotAKnownHTMLTag1) {
   1445   const char *Source = "// <tag>";
   1446 
   1447   std::vector<Token> Toks;
   1448 
   1449   lexString(Source, Toks);
   1450 
   1451   ASSERT_EQ(4U, Toks.size());
   1452 
   1453   ASSERT_EQ(tok::text,         Toks[0].getKind());
   1454   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
   1455 
   1456   ASSERT_EQ(tok::text,         Toks[1].getKind());
   1457   ASSERT_EQ(StringRef("<tag"), Toks[1].getText());
   1458 
   1459   ASSERT_EQ(tok::text,         Toks[2].getKind());
   1460   ASSERT_EQ(StringRef(">"),    Toks[2].getText());
   1461 
   1462   ASSERT_EQ(tok::newline,      Toks[3].getKind());
   1463 }
   1464 
   1465 TEST_F(CommentLexerTest, NotAKnownHTMLTag2) {
   1466   const char *Source = "// </tag>";
   1467 
   1468   std::vector<Token> Toks;
   1469 
   1470   lexString(Source, Toks);
   1471 
   1472   ASSERT_EQ(4U, Toks.size());
   1473 
   1474   ASSERT_EQ(tok::text,          Toks[0].getKind());
   1475   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
   1476 
   1477   ASSERT_EQ(tok::text,          Toks[1].getKind());
   1478   ASSERT_EQ(StringRef("</tag"), Toks[1].getText());
   1479 
   1480   ASSERT_EQ(tok::text,          Toks[2].getKind());
   1481   ASSERT_EQ(StringRef(">"),     Toks[2].getText());
   1482 
   1483   ASSERT_EQ(tok::newline,       Toks[3].getKind());
   1484 }
   1485 
   1486 TEST_F(CommentLexerTest, HTMLCharacterReferences1) {
   1487   const char *Source = "// &";
   1488 
   1489   std::vector<Token> Toks;
   1490 
   1491   lexString(Source, Toks);
   1492 
   1493   ASSERT_EQ(3U, Toks.size());
   1494 
   1495   ASSERT_EQ(tok::text,         Toks[0].getKind());
   1496   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
   1497 
   1498   ASSERT_EQ(tok::text,         Toks[1].getKind());
   1499   ASSERT_EQ(StringRef("&"),    Toks[1].getText());
   1500 
   1501   ASSERT_EQ(tok::newline,      Toks[2].getKind());
   1502 }
   1503 
   1504 TEST_F(CommentLexerTest, HTMLCharacterReferences2) {
   1505   const char *Source = "// &!";
   1506 
   1507   std::vector<Token> Toks;
   1508 
   1509   lexString(Source, Toks);
   1510 
   1511   ASSERT_EQ(4U, Toks.size());
   1512 
   1513   ASSERT_EQ(tok::text,         Toks[0].getKind());
   1514   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
   1515 
   1516   ASSERT_EQ(tok::text,         Toks[1].getKind());
   1517   ASSERT_EQ(StringRef("&"),    Toks[1].getText());
   1518 
   1519   ASSERT_EQ(tok::text,         Toks[2].getKind());
   1520   ASSERT_EQ(StringRef("!"),    Toks[2].getText());
   1521 
   1522   ASSERT_EQ(tok::newline,      Toks[3].getKind());
   1523 }
   1524 
   1525 TEST_F(CommentLexerTest, HTMLCharacterReferences3) {
   1526   const char *Source = "// &amp";
   1527 
   1528   std::vector<Token> Toks;
   1529 
   1530   lexString(Source, Toks);
   1531 
   1532   ASSERT_EQ(3U, Toks.size());
   1533 
   1534   ASSERT_EQ(tok::text,         Toks[0].getKind());
   1535   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
   1536 
   1537   ASSERT_EQ(tok::text,         Toks[1].getKind());
   1538   ASSERT_EQ(StringRef("&amp"), Toks[1].getText());
   1539 
   1540   ASSERT_EQ(tok::newline,      Toks[2].getKind());
   1541 }
   1542 
   1543 TEST_F(CommentLexerTest, HTMLCharacterReferences4) {
   1544   const char *Source = "// &amp!";
   1545 
   1546   std::vector<Token> Toks;
   1547 
   1548   lexString(Source, Toks);
   1549 
   1550   ASSERT_EQ(4U, Toks.size());
   1551 
   1552   ASSERT_EQ(tok::text,         Toks[0].getKind());
   1553   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
   1554 
   1555   ASSERT_EQ(tok::text,         Toks[1].getKind());
   1556   ASSERT_EQ(StringRef("&amp"), Toks[1].getText());
   1557 
   1558   ASSERT_EQ(tok::text,         Toks[2].getKind());
   1559   ASSERT_EQ(StringRef("!"),    Toks[2].getText());
   1560 
   1561   ASSERT_EQ(tok::newline,      Toks[3].getKind());
   1562 }
   1563 
   1564 TEST_F(CommentLexerTest, HTMLCharacterReferences5) {
   1565   const char *Source = "// &#";
   1566 
   1567   std::vector<Token> Toks;
   1568 
   1569   lexString(Source, Toks);
   1570 
   1571   ASSERT_EQ(3U, Toks.size());
   1572 
   1573   ASSERT_EQ(tok::text,         Toks[0].getKind());
   1574   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
   1575 
   1576   ASSERT_EQ(tok::text,         Toks[1].getKind());
   1577   ASSERT_EQ(StringRef("&#"),   Toks[1].getText());
   1578 
   1579   ASSERT_EQ(tok::newline,      Toks[2].getKind());
   1580 }
   1581 
   1582 TEST_F(CommentLexerTest, HTMLCharacterReferences6) {
   1583   const char *Source = "// &#a";
   1584 
   1585   std::vector<Token> Toks;
   1586 
   1587   lexString(Source, Toks);
   1588 
   1589   ASSERT_EQ(4U, Toks.size());
   1590 
   1591   ASSERT_EQ(tok::text,         Toks[0].getKind());
   1592   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
   1593 
   1594   ASSERT_EQ(tok::text,         Toks[1].getKind());
   1595   ASSERT_EQ(StringRef("&#"),   Toks[1].getText());
   1596 
   1597   ASSERT_EQ(tok::text,         Toks[2].getKind());
   1598   ASSERT_EQ(StringRef("a"),    Toks[2].getText());
   1599 
   1600   ASSERT_EQ(tok::newline,      Toks[3].getKind());
   1601 }
   1602 
   1603 TEST_F(CommentLexerTest, HTMLCharacterReferences7) {
   1604   const char *Source = "// &#42";
   1605 
   1606   std::vector<Token> Toks;
   1607 
   1608   lexString(Source, Toks);
   1609 
   1610   ASSERT_EQ(3U, Toks.size());
   1611 
   1612   ASSERT_EQ(tok::text,         Toks[0].getKind());
   1613   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
   1614 
   1615   ASSERT_EQ(tok::text,         Toks[1].getKind());
   1616   ASSERT_EQ(StringRef("&#42"), Toks[1].getText());
   1617 
   1618   ASSERT_EQ(tok::newline,      Toks[2].getKind());
   1619 }
   1620 
   1621 TEST_F(CommentLexerTest, HTMLCharacterReferences8) {
   1622   const char *Source = "// &#42a";
   1623 
   1624   std::vector<Token> Toks;
   1625 
   1626   lexString(Source, Toks);
   1627 
   1628   ASSERT_EQ(4U, Toks.size());
   1629 
   1630   ASSERT_EQ(tok::text,         Toks[0].getKind());
   1631   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
   1632 
   1633   ASSERT_EQ(tok::text,         Toks[1].getKind());
   1634   ASSERT_EQ(StringRef("&#42"), Toks[1].getText());
   1635 
   1636   ASSERT_EQ(tok::text,         Toks[2].getKind());
   1637   ASSERT_EQ(StringRef("a"),    Toks[2].getText());
   1638 
   1639   ASSERT_EQ(tok::newline,      Toks[3].getKind());
   1640 }
   1641 
   1642 TEST_F(CommentLexerTest, HTMLCharacterReferences9) {
   1643   const char *Source = "// &#x";
   1644 
   1645   std::vector<Token> Toks;
   1646 
   1647   lexString(Source, Toks);
   1648 
   1649   ASSERT_EQ(3U, Toks.size());
   1650 
   1651   ASSERT_EQ(tok::text,         Toks[0].getKind());
   1652   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
   1653 
   1654   ASSERT_EQ(tok::text,         Toks[1].getKind());
   1655   ASSERT_EQ(StringRef("&#x"),  Toks[1].getText());
   1656 
   1657   ASSERT_EQ(tok::newline,      Toks[2].getKind());
   1658 }
   1659 
   1660 TEST_F(CommentLexerTest, HTMLCharacterReferences10) {
   1661   const char *Source = "// &#xz";
   1662 
   1663   std::vector<Token> Toks;
   1664 
   1665   lexString(Source, Toks);
   1666 
   1667   ASSERT_EQ(4U, Toks.size());
   1668 
   1669   ASSERT_EQ(tok::text,         Toks[0].getKind());
   1670   ASSERT_EQ(StringRef(" "),    Toks[0].getText());
   1671 
   1672   ASSERT_EQ(tok::text,         Toks[1].getKind());
   1673   ASSERT_EQ(StringRef("&#x"),  Toks[1].getText());
   1674 
   1675   ASSERT_EQ(tok::text,         Toks[2].getKind());
   1676   ASSERT_EQ(StringRef("z"),    Toks[2].getText());
   1677 
   1678   ASSERT_EQ(tok::newline,      Toks[3].getKind());
   1679 }
   1680 
   1681 TEST_F(CommentLexerTest, HTMLCharacterReferences11) {
   1682   const char *Source = "// &#xab";
   1683 
   1684   std::vector<Token> Toks;
   1685 
   1686   lexString(Source, Toks);
   1687 
   1688   ASSERT_EQ(3U, Toks.size());
   1689 
   1690   ASSERT_EQ(tok::text,          Toks[0].getKind());
   1691   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
   1692 
   1693   ASSERT_EQ(tok::text,          Toks[1].getKind());
   1694   ASSERT_EQ(StringRef("&#xab"), Toks[1].getText());
   1695 
   1696   ASSERT_EQ(tok::newline,       Toks[2].getKind());
   1697 }
   1698 
   1699 TEST_F(CommentLexerTest, HTMLCharacterReferences12) {
   1700   const char *Source = "// &#xaBz";
   1701 
   1702   std::vector<Token> Toks;
   1703 
   1704   lexString(Source, Toks);
   1705 
   1706   ASSERT_EQ(4U, Toks.size());
   1707 
   1708   ASSERT_EQ(tok::text,          Toks[0].getKind());
   1709   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
   1710 
   1711   ASSERT_EQ(tok::text,          Toks[1].getKind());
   1712   ASSERT_EQ(StringRef("&#xaB"), Toks[1].getText());
   1713 
   1714   ASSERT_EQ(tok::text,          Toks[2].getKind());
   1715   ASSERT_EQ(StringRef("z"),     Toks[2].getText());
   1716 
   1717   ASSERT_EQ(tok::newline,       Toks[3].getKind());
   1718 }
   1719 
   1720 TEST_F(CommentLexerTest, HTMLCharacterReferences13) {
   1721   const char *Source = "// &amp;";
   1722 
   1723   std::vector<Token> Toks;
   1724 
   1725   lexString(Source, Toks);
   1726 
   1727   ASSERT_EQ(3U, Toks.size());
   1728 
   1729   ASSERT_EQ(tok::text,          Toks[0].getKind());
   1730   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
   1731 
   1732   ASSERT_EQ(tok::text,          Toks[1].getKind());
   1733   ASSERT_EQ(StringRef("&"),     Toks[1].getText());
   1734 
   1735   ASSERT_EQ(tok::newline,       Toks[2].getKind());
   1736 }
   1737 
   1738 TEST_F(CommentLexerTest, HTMLCharacterReferences14) {
   1739   const char *Source = "// &amp;&lt;";
   1740 
   1741   std::vector<Token> Toks;
   1742 
   1743   lexString(Source, Toks);
   1744 
   1745   ASSERT_EQ(4U, Toks.size());
   1746 
   1747   ASSERT_EQ(tok::text,          Toks[0].getKind());
   1748   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
   1749 
   1750   ASSERT_EQ(tok::text,          Toks[1].getKind());
   1751   ASSERT_EQ(StringRef("&"),     Toks[1].getText());
   1752 
   1753   ASSERT_EQ(tok::text,          Toks[2].getKind());
   1754   ASSERT_EQ(StringRef("<"),     Toks[2].getText());
   1755 
   1756   ASSERT_EQ(tok::newline,       Toks[3].getKind());
   1757 }
   1758 
   1759 TEST_F(CommentLexerTest, HTMLCharacterReferences15) {
   1760   const char *Source = "// &amp; meow";
   1761 
   1762   std::vector<Token> Toks;
   1763 
   1764   lexString(Source, Toks);
   1765 
   1766   ASSERT_EQ(4U, Toks.size());
   1767 
   1768   ASSERT_EQ(tok::text,          Toks[0].getKind());
   1769   ASSERT_EQ(StringRef(" "),     Toks[0].getText());
   1770 
   1771   ASSERT_EQ(tok::text,          Toks[1].getKind());
   1772   ASSERT_EQ(StringRef("&"),     Toks[1].getText());
   1773 
   1774   ASSERT_EQ(tok::text,          Toks[2].getKind());
   1775   ASSERT_EQ(StringRef(" meow"), Toks[2].getText());
   1776 
   1777   ASSERT_EQ(tok::newline,       Toks[3].getKind());
   1778 }
   1779 
   1780 TEST_F(CommentLexerTest, HTMLCharacterReferences16) {
   1781   const char *Sources[] = {
   1782     "// &#61;",
   1783     "// &#x3d;",
   1784     "// &#X3d;",
   1785     "// &#X3D;"
   1786   };
   1787 
   1788   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
   1789     std::vector<Token> Toks;
   1790 
   1791     lexString(Sources[i], Toks);
   1792 
   1793     ASSERT_EQ(3U, Toks.size());
   1794 
   1795     ASSERT_EQ(tok::text,          Toks[0].getKind());
   1796     ASSERT_EQ(StringRef(" "),     Toks[0].getText());
   1797 
   1798     ASSERT_EQ(tok::text,          Toks[1].getKind());
   1799     ASSERT_EQ(StringRef("="),     Toks[1].getText());
   1800 
   1801     ASSERT_EQ(tok::newline,       Toks[2].getKind());
   1802   }
   1803 }
   1804 
   1805 TEST_F(CommentLexerTest, MultipleComments) {
   1806   const char *Source =
   1807     "// Aaa\n"
   1808     "/// Bbb\n"
   1809     "/* Ccc\n"
   1810     " * Ddd*/\n"
   1811     "/** Eee*/";
   1812 
   1813   std::vector<Token> Toks;
   1814 
   1815   lexString(Source, Toks);
   1816 
   1817   ASSERT_EQ(12U, Toks.size());
   1818 
   1819   ASSERT_EQ(tok::text,           Toks[0].getKind());
   1820   ASSERT_EQ(StringRef(" Aaa"),   Toks[0].getText());
   1821   ASSERT_EQ(tok::newline,        Toks[1].getKind());
   1822 
   1823   ASSERT_EQ(tok::text,           Toks[2].getKind());
   1824   ASSERT_EQ(StringRef(" Bbb"),   Toks[2].getText());
   1825   ASSERT_EQ(tok::newline,        Toks[3].getKind());
   1826 
   1827   ASSERT_EQ(tok::text,           Toks[4].getKind());
   1828   ASSERT_EQ(StringRef(" Ccc"),   Toks[4].getText());
   1829   ASSERT_EQ(tok::newline,        Toks[5].getKind());
   1830 
   1831   ASSERT_EQ(tok::text,           Toks[6].getKind());
   1832   ASSERT_EQ(StringRef(" Ddd"),   Toks[6].getText());
   1833   ASSERT_EQ(tok::newline,        Toks[7].getKind());
   1834   ASSERT_EQ(tok::newline,        Toks[8].getKind());
   1835 
   1836   ASSERT_EQ(tok::text,           Toks[9].getKind());
   1837   ASSERT_EQ(StringRef(" Eee"),   Toks[9].getText());
   1838 
   1839   ASSERT_EQ(tok::newline,        Toks[10].getKind());
   1840   ASSERT_EQ(tok::newline,        Toks[11].getKind());
   1841 }
   1842 
   1843 } // end namespace comments
   1844 } // end namespace clang
   1845 
   1846