1 //===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 10 #include "clang/Lex/Lexer.h" 11 #include "clang/Basic/Diagnostic.h" 12 #include "clang/Basic/DiagnosticOptions.h" 13 #include "clang/Basic/FileManager.h" 14 #include "clang/Basic/LangOptions.h" 15 #include "clang/Basic/SourceManager.h" 16 #include "clang/Basic/TargetInfo.h" 17 #include "clang/Basic/TargetOptions.h" 18 #include "clang/Lex/HeaderSearch.h" 19 #include "clang/Lex/HeaderSearchOptions.h" 20 #include "clang/Lex/ModuleLoader.h" 21 #include "clang/Lex/Preprocessor.h" 22 #include "clang/Lex/PreprocessorOptions.h" 23 #include "gtest/gtest.h" 24 25 using namespace llvm; 26 using namespace clang; 27 28 namespace { 29 30 class VoidModuleLoader : public ModuleLoader { 31 ModuleLoadResult loadModule(SourceLocation ImportLoc, 32 ModuleIdPath Path, 33 Module::NameVisibilityKind Visibility, 34 bool IsInclusionDirective) override { 35 return ModuleLoadResult(); 36 } 37 38 void makeModuleVisible(Module *Mod, 39 Module::NameVisibilityKind Visibility, 40 SourceLocation ImportLoc, 41 bool Complain) override { } 42 43 GlobalModuleIndex *loadGlobalModuleIndex(SourceLocation TriggerLoc) override 44 { return nullptr; } 45 bool lookupMissingImports(StringRef Name, SourceLocation TriggerLoc) override 46 { return 0; }; 47 }; 48 49 // The test fixture. 50 class LexerTest : public ::testing::Test { 51 protected: 52 LexerTest() 53 : FileMgr(FileMgrOpts), 54 DiagID(new DiagnosticIDs()), 55 Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()), 56 SourceMgr(Diags, FileMgr), 57 TargetOpts(new TargetOptions) 58 { 59 TargetOpts->Triple = "x86_64-apple-darwin11.1.0"; 60 Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts); 61 } 62 63 std::vector<Token> CheckLex(StringRef Source, 64 ArrayRef<tok::TokenKind> ExpectedTokens) { 65 MemoryBuffer *buf = MemoryBuffer::getMemBuffer(Source); 66 SourceMgr.setMainFileID(SourceMgr.createFileID(buf)); 67 68 VoidModuleLoader ModLoader; 69 HeaderSearch HeaderInfo(new HeaderSearchOptions, SourceMgr, Diags, LangOpts, 70 Target.get()); 71 Preprocessor PP(new PreprocessorOptions(), Diags, LangOpts, SourceMgr, 72 HeaderInfo, ModLoader, /*IILookup =*/nullptr, 73 /*OwnsHeaderSearch =*/false); 74 PP.Initialize(*Target); 75 PP.EnterMainSourceFile(); 76 77 std::vector<Token> toks; 78 while (1) { 79 Token tok; 80 PP.Lex(tok); 81 if (tok.is(tok::eof)) 82 break; 83 toks.push_back(tok); 84 } 85 86 EXPECT_EQ(ExpectedTokens.size(), toks.size()); 87 for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) { 88 EXPECT_EQ(ExpectedTokens[i], toks[i].getKind()); 89 } 90 91 return toks; 92 } 93 94 std::string getSourceText(Token Begin, Token End) { 95 bool Invalid; 96 StringRef Str = 97 Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange( 98 Begin.getLocation(), End.getLocation())), 99 SourceMgr, LangOpts, &Invalid); 100 if (Invalid) 101 return "<INVALID>"; 102 return Str; 103 } 104 105 FileSystemOptions FileMgrOpts; 106 FileManager FileMgr; 107 IntrusiveRefCntPtr<DiagnosticIDs> DiagID; 108 DiagnosticsEngine Diags; 109 SourceManager SourceMgr; 110 LangOptions LangOpts; 111 std::shared_ptr<TargetOptions> TargetOpts; 112 IntrusiveRefCntPtr<TargetInfo> Target; 113 }; 114 115 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) { 116 std::vector<tok::TokenKind> ExpectedTokens; 117 ExpectedTokens.push_back(tok::identifier); 118 ExpectedTokens.push_back(tok::l_paren); 119 ExpectedTokens.push_back(tok::identifier); 120 ExpectedTokens.push_back(tok::r_paren); 121 122 std::vector<Token> toks = CheckLex("#define M(x) x\n" 123 "M(f(M(i)))", 124 ExpectedTokens); 125 126 EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2])); 127 } 128 129 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) { 130 std::vector<tok::TokenKind> ExpectedTokens; 131 ExpectedTokens.push_back(tok::identifier); 132 ExpectedTokens.push_back(tok::identifier); 133 134 std::vector<Token> toks = CheckLex("#define M(x) x\n" 135 "M(M(i) c)", 136 ExpectedTokens); 137 138 EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0])); 139 } 140 141 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) { 142 std::vector<tok::TokenKind> ExpectedTokens; 143 ExpectedTokens.push_back(tok::identifier); 144 ExpectedTokens.push_back(tok::identifier); 145 ExpectedTokens.push_back(tok::identifier); 146 147 std::vector<Token> toks = CheckLex("#define M(x) x\n" 148 "M(c c M(i))", 149 ExpectedTokens); 150 151 EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2])); 152 } 153 154 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) { 155 std::vector<tok::TokenKind> ExpectedTokens; 156 ExpectedTokens.push_back(tok::identifier); 157 ExpectedTokens.push_back(tok::identifier); 158 ExpectedTokens.push_back(tok::identifier); 159 160 std::vector<Token> toks = CheckLex("#define M(x) x\n" 161 "M(M(i) c c)", 162 ExpectedTokens); 163 164 EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1])); 165 } 166 167 TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) { 168 std::vector<tok::TokenKind> ExpectedTokens; 169 ExpectedTokens.push_back(tok::identifier); 170 ExpectedTokens.push_back(tok::identifier); 171 ExpectedTokens.push_back(tok::identifier); 172 ExpectedTokens.push_back(tok::identifier); 173 174 std::vector<Token> toks = CheckLex("#define M(x) x\n" 175 "M(c M(i)) M(M(i) c)", 176 ExpectedTokens); 177 178 EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2])); 179 } 180 181 TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) { 182 std::vector<tok::TokenKind> ExpectedTokens; 183 ExpectedTokens.push_back(tok::identifier); 184 ExpectedTokens.push_back(tok::l_paren); 185 ExpectedTokens.push_back(tok::identifier); 186 ExpectedTokens.push_back(tok::r_paren); 187 188 std::vector<Token> toks = CheckLex("#define M(x) x\n" 189 "#define C(x) M(x##c)\n" 190 "M(f(C(i)))", 191 ExpectedTokens); 192 193 EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2])); 194 } 195 196 TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) { 197 std::vector<tok::TokenKind> ExpectedTokens; 198 ExpectedTokens.push_back(tok::identifier); 199 ExpectedTokens.push_back(tok::l_paren); 200 ExpectedTokens.push_back(tok::identifier); 201 ExpectedTokens.push_back(tok::r_paren); 202 203 std::vector<Token> toks = CheckLex("#define M(x) x\n" 204 "f(M(M(i)))", 205 ExpectedTokens); 206 EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2])); 207 } 208 209 TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) { 210 std::vector<tok::TokenKind> ExpectedTokens; 211 ExpectedTokens.push_back(tok::identifier); 212 ExpectedTokens.push_back(tok::l_paren); 213 ExpectedTokens.push_back(tok::identifier); 214 ExpectedTokens.push_back(tok::r_paren); 215 216 std::vector<Token> toks = CheckLex("#define M(x) x\n" 217 "M(f(i))", 218 ExpectedTokens); 219 EXPECT_EQ("i", getSourceText(toks[2], toks[2])); 220 } 221 222 TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) { 223 std::vector<tok::TokenKind> ExpectedTokens; 224 ExpectedTokens.push_back(tok::identifier); 225 ExpectedTokens.push_back(tok::l_paren); 226 ExpectedTokens.push_back(tok::identifier); 227 ExpectedTokens.push_back(tok::r_paren); 228 229 std::vector<Token> toks = CheckLex("#define M(x) x\n" 230 "#define C(x) x\n" 231 "f(C(M(i)))", 232 ExpectedTokens); 233 EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2])); 234 } 235 236 TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) { 237 std::vector<tok::TokenKind> ExpectedTokens; 238 ExpectedTokens.push_back(tok::identifier); 239 ExpectedTokens.push_back(tok::l_paren); 240 ExpectedTokens.push_back(tok::identifier); 241 ExpectedTokens.push_back(tok::identifier); 242 ExpectedTokens.push_back(tok::r_paren); 243 244 std::vector<Token> toks = CheckLex("#define M(x) x\n" 245 "#define C(x) c x\n" 246 "f(C(M(i)))", 247 ExpectedTokens); 248 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3])); 249 } 250 251 TEST_F(LexerTest, GetSourceTextExpandsRecursively) { 252 std::vector<tok::TokenKind> ExpectedTokens; 253 ExpectedTokens.push_back(tok::identifier); 254 ExpectedTokens.push_back(tok::identifier); 255 ExpectedTokens.push_back(tok::l_paren); 256 ExpectedTokens.push_back(tok::identifier); 257 ExpectedTokens.push_back(tok::r_paren); 258 259 std::vector<Token> toks = CheckLex("#define M(x) x\n" 260 "#define C(x) c M(x)\n" 261 "C(f(M(i)))", 262 ExpectedTokens); 263 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3])); 264 } 265 266 TEST_F(LexerTest, LexAPI) { 267 std::vector<tok::TokenKind> ExpectedTokens; 268 ExpectedTokens.push_back(tok::l_square); 269 ExpectedTokens.push_back(tok::identifier); 270 ExpectedTokens.push_back(tok::r_square); 271 ExpectedTokens.push_back(tok::l_square); 272 ExpectedTokens.push_back(tok::identifier); 273 ExpectedTokens.push_back(tok::r_square); 274 ExpectedTokens.push_back(tok::identifier); 275 ExpectedTokens.push_back(tok::identifier); 276 ExpectedTokens.push_back(tok::identifier); 277 ExpectedTokens.push_back(tok::identifier); 278 279 std::vector<Token> toks = CheckLex("#define M(x) [x]\n" 280 "#define N(x) x\n" 281 "#define INN(x) x\n" 282 "#define NOF1 INN(val)\n" 283 "#define NOF2 val\n" 284 "M(foo) N([bar])\n" 285 "N(INN(val)) N(NOF1) N(NOF2) N(val)", 286 ExpectedTokens); 287 288 SourceLocation lsqrLoc = toks[0].getLocation(); 289 SourceLocation idLoc = toks[1].getLocation(); 290 SourceLocation rsqrLoc = toks[2].getLocation(); 291 std::pair<SourceLocation,SourceLocation> 292 macroPair = SourceMgr.getExpansionRange(lsqrLoc); 293 SourceRange macroRange = SourceRange(macroPair.first, macroPair.second); 294 295 SourceLocation Loc; 296 EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc)); 297 EXPECT_EQ(Loc, macroRange.getBegin()); 298 EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts)); 299 EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts)); 300 EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc)); 301 EXPECT_EQ(Loc, macroRange.getEnd()); 302 303 CharSourceRange range = Lexer::makeFileCharRange( 304 CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts); 305 EXPECT_TRUE(range.isInvalid()); 306 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc), 307 SourceMgr, LangOpts); 308 EXPECT_TRUE(range.isInvalid()); 309 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc), 310 SourceMgr, LangOpts); 311 EXPECT_TRUE(!range.isTokenRange()); 312 EXPECT_EQ(range.getAsRange(), 313 SourceRange(macroRange.getBegin(), 314 macroRange.getEnd().getLocWithOffset(1))); 315 316 StringRef text = Lexer::getSourceText( 317 CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc), 318 SourceMgr, LangOpts); 319 EXPECT_EQ(text, "M(foo)"); 320 321 SourceLocation macroLsqrLoc = toks[3].getLocation(); 322 SourceLocation macroIdLoc = toks[4].getLocation(); 323 SourceLocation macroRsqrLoc = toks[5].getLocation(); 324 SourceLocation fileLsqrLoc = SourceMgr.getSpellingLoc(macroLsqrLoc); 325 SourceLocation fileIdLoc = SourceMgr.getSpellingLoc(macroIdLoc); 326 SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc); 327 328 range = Lexer::makeFileCharRange( 329 CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc), 330 SourceMgr, LangOpts); 331 EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)), 332 range.getAsRange()); 333 334 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc), 335 SourceMgr, LangOpts); 336 EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)), 337 range.getAsRange()); 338 339 macroPair = SourceMgr.getExpansionRange(macroLsqrLoc); 340 range = Lexer::makeFileCharRange( 341 CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc), 342 SourceMgr, LangOpts); 343 EXPECT_EQ(SourceRange(macroPair.first, macroPair.second.getLocWithOffset(1)), 344 range.getAsRange()); 345 346 text = Lexer::getSourceText( 347 CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)), 348 SourceMgr, LangOpts); 349 EXPECT_EQ(text, "[bar"); 350 351 352 SourceLocation idLoc1 = toks[6].getLocation(); 353 SourceLocation idLoc2 = toks[7].getLocation(); 354 SourceLocation idLoc3 = toks[8].getLocation(); 355 SourceLocation idLoc4 = toks[9].getLocation(); 356 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc1, SourceMgr, LangOpts)); 357 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc2, SourceMgr, LangOpts)); 358 EXPECT_EQ("NOF2", Lexer::getImmediateMacroName(idLoc3, SourceMgr, LangOpts)); 359 EXPECT_EQ("N", Lexer::getImmediateMacroName(idLoc4, SourceMgr, LangOpts)); 360 } 361 362 } // anonymous namespace 363
