1 //===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 10 #include "clang/Lex/Lexer.h" 11 #include "clang/Basic/Diagnostic.h" 12 #include "clang/Basic/DiagnosticOptions.h" 13 #include "clang/Basic/FileManager.h" 14 #include "clang/Basic/LangOptions.h" 15 #include "clang/Basic/SourceManager.h" 16 #include "clang/Basic/TargetInfo.h" 17 #include "clang/Basic/TargetOptions.h" 18 #include "clang/Lex/HeaderSearch.h" 19 #include "clang/Lex/HeaderSearchOptions.h" 20 #include "clang/Lex/ModuleLoader.h" 21 #include "clang/Lex/Preprocessor.h" 22 #include "clang/Lex/PreprocessorOptions.h" 23 #include "gtest/gtest.h" 24 25 using namespace llvm; 26 using namespace clang; 27 28 namespace { 29 30 class VoidModuleLoader : public ModuleLoader { 31 ModuleLoadResult loadModule(SourceLocation ImportLoc, 32 ModuleIdPath Path, 33 Module::NameVisibilityKind Visibility, 34 bool IsInclusionDirective) override { 35 return ModuleLoadResult(); 36 } 37 38 void makeModuleVisible(Module *Mod, 39 Module::NameVisibilityKind Visibility, 40 SourceLocation ImportLoc) override { } 41 42 GlobalModuleIndex *loadGlobalModuleIndex(SourceLocation TriggerLoc) override 43 { return nullptr; } 44 bool lookupMissingImports(StringRef Name, SourceLocation TriggerLoc) override 45 { return 0; } 46 }; 47 48 // The test fixture. 49 class LexerTest : public ::testing::Test { 50 protected: 51 LexerTest() 52 : FileMgr(FileMgrOpts), 53 DiagID(new DiagnosticIDs()), 54 Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()), 55 SourceMgr(Diags, FileMgr), 56 TargetOpts(new TargetOptions) 57 { 58 TargetOpts->Triple = "x86_64-apple-darwin11.1.0"; 59 Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts); 60 } 61 62 std::vector<Token> CheckLex(StringRef Source, 63 ArrayRef<tok::TokenKind> ExpectedTokens) { 64 std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Source); 65 SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf))); 66 67 VoidModuleLoader ModLoader; 68 HeaderSearch HeaderInfo(new HeaderSearchOptions, SourceMgr, Diags, LangOpts, 69 Target.get()); 70 Preprocessor PP(new PreprocessorOptions(), Diags, LangOpts, SourceMgr, 71 HeaderInfo, ModLoader, /*IILookup =*/nullptr, 72 /*OwnsHeaderSearch =*/false); 73 PP.Initialize(*Target); 74 PP.EnterMainSourceFile(); 75 76 std::vector<Token> toks; 77 while (1) { 78 Token tok; 79 PP.Lex(tok); 80 if (tok.is(tok::eof)) 81 break; 82 toks.push_back(tok); 83 } 84 85 EXPECT_EQ(ExpectedTokens.size(), toks.size()); 86 for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) { 87 EXPECT_EQ(ExpectedTokens[i], toks[i].getKind()); 88 } 89 90 return toks; 91 } 92 93 std::string getSourceText(Token Begin, Token End) { 94 bool Invalid; 95 StringRef Str = 96 Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange( 97 Begin.getLocation(), End.getLocation())), 98 SourceMgr, LangOpts, &Invalid); 99 if (Invalid) 100 return "<INVALID>"; 101 return Str; 102 } 103 104 FileSystemOptions FileMgrOpts; 105 FileManager FileMgr; 106 IntrusiveRefCntPtr<DiagnosticIDs> DiagID; 107 DiagnosticsEngine Diags; 108 SourceManager SourceMgr; 109 LangOptions LangOpts; 110 std::shared_ptr<TargetOptions> TargetOpts; 111 IntrusiveRefCntPtr<TargetInfo> Target; 112 }; 113 114 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) { 115 std::vector<tok::TokenKind> ExpectedTokens; 116 ExpectedTokens.push_back(tok::identifier); 117 ExpectedTokens.push_back(tok::l_paren); 118 ExpectedTokens.push_back(tok::identifier); 119 ExpectedTokens.push_back(tok::r_paren); 120 121 std::vector<Token> toks = CheckLex("#define M(x) x\n" 122 "M(f(M(i)))", 123 ExpectedTokens); 124 125 EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2])); 126 } 127 128 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) { 129 std::vector<tok::TokenKind> ExpectedTokens; 130 ExpectedTokens.push_back(tok::identifier); 131 ExpectedTokens.push_back(tok::identifier); 132 133 std::vector<Token> toks = CheckLex("#define M(x) x\n" 134 "M(M(i) c)", 135 ExpectedTokens); 136 137 EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0])); 138 } 139 140 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) { 141 std::vector<tok::TokenKind> ExpectedTokens; 142 ExpectedTokens.push_back(tok::identifier); 143 ExpectedTokens.push_back(tok::identifier); 144 ExpectedTokens.push_back(tok::identifier); 145 146 std::vector<Token> toks = CheckLex("#define M(x) x\n" 147 "M(c c M(i))", 148 ExpectedTokens); 149 150 EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2])); 151 } 152 153 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) { 154 std::vector<tok::TokenKind> ExpectedTokens; 155 ExpectedTokens.push_back(tok::identifier); 156 ExpectedTokens.push_back(tok::identifier); 157 ExpectedTokens.push_back(tok::identifier); 158 159 std::vector<Token> toks = CheckLex("#define M(x) x\n" 160 "M(M(i) c c)", 161 ExpectedTokens); 162 163 EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1])); 164 } 165 166 TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) { 167 std::vector<tok::TokenKind> ExpectedTokens; 168 ExpectedTokens.push_back(tok::identifier); 169 ExpectedTokens.push_back(tok::identifier); 170 ExpectedTokens.push_back(tok::identifier); 171 ExpectedTokens.push_back(tok::identifier); 172 173 std::vector<Token> toks = CheckLex("#define M(x) x\n" 174 "M(c M(i)) M(M(i) c)", 175 ExpectedTokens); 176 177 EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2])); 178 } 179 180 TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) { 181 std::vector<tok::TokenKind> ExpectedTokens; 182 ExpectedTokens.push_back(tok::identifier); 183 ExpectedTokens.push_back(tok::l_paren); 184 ExpectedTokens.push_back(tok::identifier); 185 ExpectedTokens.push_back(tok::r_paren); 186 187 std::vector<Token> toks = CheckLex("#define M(x) x\n" 188 "#define C(x) M(x##c)\n" 189 "M(f(C(i)))", 190 ExpectedTokens); 191 192 EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2])); 193 } 194 195 TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) { 196 std::vector<tok::TokenKind> ExpectedTokens; 197 ExpectedTokens.push_back(tok::identifier); 198 ExpectedTokens.push_back(tok::l_paren); 199 ExpectedTokens.push_back(tok::identifier); 200 ExpectedTokens.push_back(tok::r_paren); 201 202 std::vector<Token> toks = CheckLex("#define M(x) x\n" 203 "f(M(M(i)))", 204 ExpectedTokens); 205 EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2])); 206 } 207 208 TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) { 209 std::vector<tok::TokenKind> ExpectedTokens; 210 ExpectedTokens.push_back(tok::identifier); 211 ExpectedTokens.push_back(tok::l_paren); 212 ExpectedTokens.push_back(tok::identifier); 213 ExpectedTokens.push_back(tok::r_paren); 214 215 std::vector<Token> toks = CheckLex("#define M(x) x\n" 216 "M(f(i))", 217 ExpectedTokens); 218 EXPECT_EQ("i", getSourceText(toks[2], toks[2])); 219 } 220 221 TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) { 222 std::vector<tok::TokenKind> ExpectedTokens; 223 ExpectedTokens.push_back(tok::identifier); 224 ExpectedTokens.push_back(tok::l_paren); 225 ExpectedTokens.push_back(tok::identifier); 226 ExpectedTokens.push_back(tok::r_paren); 227 228 std::vector<Token> toks = CheckLex("#define M(x) x\n" 229 "#define C(x) x\n" 230 "f(C(M(i)))", 231 ExpectedTokens); 232 EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2])); 233 } 234 235 TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) { 236 std::vector<tok::TokenKind> ExpectedTokens; 237 ExpectedTokens.push_back(tok::identifier); 238 ExpectedTokens.push_back(tok::l_paren); 239 ExpectedTokens.push_back(tok::identifier); 240 ExpectedTokens.push_back(tok::identifier); 241 ExpectedTokens.push_back(tok::r_paren); 242 243 std::vector<Token> toks = CheckLex("#define M(x) x\n" 244 "#define C(x) c x\n" 245 "f(C(M(i)))", 246 ExpectedTokens); 247 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3])); 248 } 249 250 TEST_F(LexerTest, GetSourceTextExpandsRecursively) { 251 std::vector<tok::TokenKind> ExpectedTokens; 252 ExpectedTokens.push_back(tok::identifier); 253 ExpectedTokens.push_back(tok::identifier); 254 ExpectedTokens.push_back(tok::l_paren); 255 ExpectedTokens.push_back(tok::identifier); 256 ExpectedTokens.push_back(tok::r_paren); 257 258 std::vector<Token> toks = CheckLex("#define M(x) x\n" 259 "#define C(x) c M(x)\n" 260 "C(f(M(i)))", 261 ExpectedTokens); 262 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3])); 263 } 264 265 TEST_F(LexerTest, LexAPI) { 266 std::vector<tok::TokenKind> ExpectedTokens; 267 ExpectedTokens.push_back(tok::l_square); 268 ExpectedTokens.push_back(tok::identifier); 269 ExpectedTokens.push_back(tok::r_square); 270 ExpectedTokens.push_back(tok::l_square); 271 ExpectedTokens.push_back(tok::identifier); 272 ExpectedTokens.push_back(tok::r_square); 273 ExpectedTokens.push_back(tok::identifier); 274 ExpectedTokens.push_back(tok::identifier); 275 ExpectedTokens.push_back(tok::identifier); 276 ExpectedTokens.push_back(tok::identifier); 277 278 std::vector<Token> toks = CheckLex("#define M(x) [x]\n" 279 "#define N(x) x\n" 280 "#define INN(x) x\n" 281 "#define NOF1 INN(val)\n" 282 "#define NOF2 val\n" 283 "M(foo) N([bar])\n" 284 "N(INN(val)) N(NOF1) N(NOF2) N(val)", 285 ExpectedTokens); 286 287 SourceLocation lsqrLoc = toks[0].getLocation(); 288 SourceLocation idLoc = toks[1].getLocation(); 289 SourceLocation rsqrLoc = toks[2].getLocation(); 290 std::pair<SourceLocation,SourceLocation> 291 macroPair = SourceMgr.getExpansionRange(lsqrLoc); 292 SourceRange macroRange = SourceRange(macroPair.first, macroPair.second); 293 294 SourceLocation Loc; 295 EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc)); 296 EXPECT_EQ(Loc, macroRange.getBegin()); 297 EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts)); 298 EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts)); 299 EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc)); 300 EXPECT_EQ(Loc, macroRange.getEnd()); 301 302 CharSourceRange range = Lexer::makeFileCharRange( 303 CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts); 304 EXPECT_TRUE(range.isInvalid()); 305 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc), 306 SourceMgr, LangOpts); 307 EXPECT_TRUE(range.isInvalid()); 308 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc), 309 SourceMgr, LangOpts); 310 EXPECT_TRUE(!range.isTokenRange()); 311 EXPECT_EQ(range.getAsRange(), 312 SourceRange(macroRange.getBegin(), 313 macroRange.getEnd().getLocWithOffset(1))); 314 315 StringRef text = Lexer::getSourceText( 316 CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc), 317 SourceMgr, LangOpts); 318 EXPECT_EQ(text, "M(foo)"); 319 320 SourceLocation macroLsqrLoc = toks[3].getLocation(); 321 SourceLocation macroIdLoc = toks[4].getLocation(); 322 SourceLocation macroRsqrLoc = toks[5].getLocation(); 323 SourceLocation fileLsqrLoc = SourceMgr.getSpellingLoc(macroLsqrLoc); 324 SourceLocation fileIdLoc = SourceMgr.getSpellingLoc(macroIdLoc); 325 SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc); 326 327 range = Lexer::makeFileCharRange( 328 CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc), 329 SourceMgr, LangOpts); 330 EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)), 331 range.getAsRange()); 332 333 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc), 334 SourceMgr, LangOpts); 335 EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)), 336 range.getAsRange()); 337 338 macroPair = SourceMgr.getExpansionRange(macroLsqrLoc); 339 range = Lexer::makeFileCharRange( 340 CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc), 341 SourceMgr, LangOpts); 342 EXPECT_EQ(SourceRange(macroPair.first, macroPair.second.getLocWithOffset(1)), 343 range.getAsRange()); 344 345 text = Lexer::getSourceText( 346 CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)), 347 SourceMgr, LangOpts); 348 EXPECT_EQ(text, "[bar"); 349 350 351 SourceLocation idLoc1 = toks[6].getLocation(); 352 SourceLocation idLoc2 = toks[7].getLocation(); 353 SourceLocation idLoc3 = toks[8].getLocation(); 354 SourceLocation idLoc4 = toks[9].getLocation(); 355 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc1, SourceMgr, LangOpts)); 356 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc2, SourceMgr, LangOpts)); 357 EXPECT_EQ("NOF2", Lexer::getImmediateMacroName(idLoc3, SourceMgr, LangOpts)); 358 EXPECT_EQ("N", Lexer::getImmediateMacroName(idLoc4, SourceMgr, LangOpts)); 359 } 360 361 } // anonymous namespace 362
