1 //===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 10 #include "clang/Lex/Lexer.h" 11 #include "clang/Basic/Diagnostic.h" 12 #include "clang/Basic/DiagnosticOptions.h" 13 #include "clang/Basic/FileManager.h" 14 #include "clang/Basic/LangOptions.h" 15 #include "clang/Basic/SourceManager.h" 16 #include "clang/Basic/TargetInfo.h" 17 #include "clang/Basic/TargetOptions.h" 18 #include "clang/Lex/HeaderSearch.h" 19 #include "clang/Lex/HeaderSearchOptions.h" 20 #include "clang/Lex/ModuleLoader.h" 21 #include "clang/Lex/Preprocessor.h" 22 #include "clang/Lex/PreprocessorOptions.h" 23 #include "llvm/Config/config.h" 24 #include "gtest/gtest.h" 25 26 using namespace llvm; 27 using namespace clang; 28 29 namespace { 30 31 class VoidModuleLoader : public ModuleLoader { 32 virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, 33 ModuleIdPath Path, 34 Module::NameVisibilityKind Visibility, 35 bool IsInclusionDirective) { 36 return ModuleLoadResult(); 37 } 38 39 virtual void makeModuleVisible(Module *Mod, 40 Module::NameVisibilityKind Visibility, 41 SourceLocation ImportLoc, 42 bool Complain) { } 43 }; 44 45 // The test fixture. 46 class LexerTest : public ::testing::Test { 47 protected: 48 LexerTest() 49 : FileMgr(FileMgrOpts), 50 DiagID(new DiagnosticIDs()), 51 Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()), 52 SourceMgr(Diags, FileMgr), 53 TargetOpts(new TargetOptions) 54 { 55 TargetOpts->Triple = "x86_64-apple-darwin11.1.0"; 56 Target = TargetInfo::CreateTargetInfo(Diags, &*TargetOpts); 57 } 58 59 std::vector<Token> CheckLex(StringRef Source, 60 ArrayRef<tok::TokenKind> ExpectedTokens) { 61 MemoryBuffer *buf = MemoryBuffer::getMemBuffer(Source); 62 (void) SourceMgr.createMainFileIDForMemBuffer(buf); 63 64 VoidModuleLoader ModLoader; 65 HeaderSearch HeaderInfo(new HeaderSearchOptions, FileMgr, Diags, LangOpts, 66 Target.getPtr()); 67 Preprocessor PP(new PreprocessorOptions(), Diags, LangOpts, Target.getPtr(), 68 SourceMgr, HeaderInfo, ModLoader, /*IILookup =*/ 0, 69 /*OwnsHeaderSearch =*/ false, 70 /*DelayInitialization =*/ false); 71 PP.EnterMainSourceFile(); 72 73 std::vector<Token> toks; 74 while (1) { 75 Token tok; 76 PP.Lex(tok); 77 if (tok.is(tok::eof)) 78 break; 79 toks.push_back(tok); 80 } 81 82 EXPECT_EQ(ExpectedTokens.size(), toks.size()); 83 for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) { 84 EXPECT_EQ(ExpectedTokens[i], toks[i].getKind()); 85 } 86 87 return toks; 88 } 89 90 std::string getSourceText(Token Begin, Token End) { 91 bool Invalid; 92 StringRef Str = 93 Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange( 94 Begin.getLocation(), End.getLocation())), 95 SourceMgr, LangOpts, &Invalid); 96 if (Invalid) 97 return "<INVALID>"; 98 return Str; 99 } 100 101 FileSystemOptions FileMgrOpts; 102 FileManager FileMgr; 103 IntrusiveRefCntPtr<DiagnosticIDs> DiagID; 104 DiagnosticsEngine Diags; 105 SourceManager SourceMgr; 106 LangOptions LangOpts; 107 IntrusiveRefCntPtr<TargetOptions> TargetOpts; 108 IntrusiveRefCntPtr<TargetInfo> Target; 109 }; 110 111 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) { 112 std::vector<tok::TokenKind> ExpectedTokens; 113 ExpectedTokens.push_back(tok::identifier); 114 ExpectedTokens.push_back(tok::l_paren); 115 ExpectedTokens.push_back(tok::identifier); 116 ExpectedTokens.push_back(tok::r_paren); 117 118 std::vector<Token> toks = CheckLex("#define M(x) x\n" 119 "M(f(M(i)))", 120 ExpectedTokens); 121 122 EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2])); 123 } 124 125 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) { 126 std::vector<tok::TokenKind> ExpectedTokens; 127 ExpectedTokens.push_back(tok::identifier); 128 ExpectedTokens.push_back(tok::identifier); 129 130 std::vector<Token> toks = CheckLex("#define M(x) x\n" 131 "M(M(i) c)", 132 ExpectedTokens); 133 134 EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0])); 135 } 136 137 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) { 138 std::vector<tok::TokenKind> ExpectedTokens; 139 ExpectedTokens.push_back(tok::identifier); 140 ExpectedTokens.push_back(tok::identifier); 141 ExpectedTokens.push_back(tok::identifier); 142 143 std::vector<Token> toks = CheckLex("#define M(x) x\n" 144 "M(c c M(i))", 145 ExpectedTokens); 146 147 EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2])); 148 } 149 150 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) { 151 std::vector<tok::TokenKind> ExpectedTokens; 152 ExpectedTokens.push_back(tok::identifier); 153 ExpectedTokens.push_back(tok::identifier); 154 ExpectedTokens.push_back(tok::identifier); 155 156 std::vector<Token> toks = CheckLex("#define M(x) x\n" 157 "M(M(i) c c)", 158 ExpectedTokens); 159 160 EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1])); 161 } 162 163 TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) { 164 std::vector<tok::TokenKind> ExpectedTokens; 165 ExpectedTokens.push_back(tok::identifier); 166 ExpectedTokens.push_back(tok::identifier); 167 ExpectedTokens.push_back(tok::identifier); 168 ExpectedTokens.push_back(tok::identifier); 169 170 std::vector<Token> toks = CheckLex("#define M(x) x\n" 171 "M(c M(i)) M(M(i) c)", 172 ExpectedTokens); 173 174 EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2])); 175 } 176 177 TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) { 178 std::vector<tok::TokenKind> ExpectedTokens; 179 ExpectedTokens.push_back(tok::identifier); 180 ExpectedTokens.push_back(tok::l_paren); 181 ExpectedTokens.push_back(tok::identifier); 182 ExpectedTokens.push_back(tok::r_paren); 183 184 std::vector<Token> toks = CheckLex("#define M(x) x\n" 185 "#define C(x) M(x##c)\n" 186 "M(f(C(i)))", 187 ExpectedTokens); 188 189 EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2])); 190 } 191 192 TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) { 193 std::vector<tok::TokenKind> ExpectedTokens; 194 ExpectedTokens.push_back(tok::identifier); 195 ExpectedTokens.push_back(tok::l_paren); 196 ExpectedTokens.push_back(tok::identifier); 197 ExpectedTokens.push_back(tok::r_paren); 198 199 std::vector<Token> toks = CheckLex("#define M(x) x\n" 200 "f(M(M(i)))", 201 ExpectedTokens); 202 EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2])); 203 } 204 205 TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) { 206 std::vector<tok::TokenKind> ExpectedTokens; 207 ExpectedTokens.push_back(tok::identifier); 208 ExpectedTokens.push_back(tok::l_paren); 209 ExpectedTokens.push_back(tok::identifier); 210 ExpectedTokens.push_back(tok::r_paren); 211 212 std::vector<Token> toks = CheckLex("#define M(x) x\n" 213 "M(f(i))", 214 ExpectedTokens); 215 EXPECT_EQ("i", getSourceText(toks[2], toks[2])); 216 } 217 218 TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) { 219 std::vector<tok::TokenKind> ExpectedTokens; 220 ExpectedTokens.push_back(tok::identifier); 221 ExpectedTokens.push_back(tok::l_paren); 222 ExpectedTokens.push_back(tok::identifier); 223 ExpectedTokens.push_back(tok::r_paren); 224 225 std::vector<Token> toks = CheckLex("#define M(x) x\n" 226 "#define C(x) x\n" 227 "f(C(M(i)))", 228 ExpectedTokens); 229 EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2])); 230 } 231 232 TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) { 233 std::vector<tok::TokenKind> ExpectedTokens; 234 ExpectedTokens.push_back(tok::identifier); 235 ExpectedTokens.push_back(tok::l_paren); 236 ExpectedTokens.push_back(tok::identifier); 237 ExpectedTokens.push_back(tok::identifier); 238 ExpectedTokens.push_back(tok::r_paren); 239 240 std::vector<Token> toks = CheckLex("#define M(x) x\n" 241 "#define C(x) c x\n" 242 "f(C(M(i)))", 243 ExpectedTokens); 244 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3])); 245 } 246 247 TEST_F(LexerTest, GetSourceTextExpandsRecursively) { 248 std::vector<tok::TokenKind> ExpectedTokens; 249 ExpectedTokens.push_back(tok::identifier); 250 ExpectedTokens.push_back(tok::identifier); 251 ExpectedTokens.push_back(tok::l_paren); 252 ExpectedTokens.push_back(tok::identifier); 253 ExpectedTokens.push_back(tok::r_paren); 254 255 std::vector<Token> toks = CheckLex("#define M(x) x\n" 256 "#define C(x) c M(x)\n" 257 "C(f(M(i)))", 258 ExpectedTokens); 259 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3])); 260 } 261 262 TEST_F(LexerTest, LexAPI) { 263 std::vector<tok::TokenKind> ExpectedTokens; 264 ExpectedTokens.push_back(tok::l_square); 265 ExpectedTokens.push_back(tok::identifier); 266 ExpectedTokens.push_back(tok::r_square); 267 ExpectedTokens.push_back(tok::l_square); 268 ExpectedTokens.push_back(tok::identifier); 269 ExpectedTokens.push_back(tok::r_square); 270 ExpectedTokens.push_back(tok::identifier); 271 ExpectedTokens.push_back(tok::identifier); 272 ExpectedTokens.push_back(tok::identifier); 273 ExpectedTokens.push_back(tok::identifier); 274 275 std::vector<Token> toks = CheckLex("#define M(x) [x]\n" 276 "#define N(x) x\n" 277 "#define INN(x) x\n" 278 "#define NOF1 INN(val)\n" 279 "#define NOF2 val\n" 280 "M(foo) N([bar])\n" 281 "N(INN(val)) N(NOF1) N(NOF2) N(val)", 282 ExpectedTokens); 283 284 SourceLocation lsqrLoc = toks[0].getLocation(); 285 SourceLocation idLoc = toks[1].getLocation(); 286 SourceLocation rsqrLoc = toks[2].getLocation(); 287 std::pair<SourceLocation,SourceLocation> 288 macroPair = SourceMgr.getExpansionRange(lsqrLoc); 289 SourceRange macroRange = SourceRange(macroPair.first, macroPair.second); 290 291 SourceLocation Loc; 292 EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc)); 293 EXPECT_EQ(Loc, macroRange.getBegin()); 294 EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts)); 295 EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts)); 296 EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc)); 297 EXPECT_EQ(Loc, macroRange.getEnd()); 298 299 CharSourceRange range = Lexer::makeFileCharRange( 300 CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts); 301 EXPECT_TRUE(range.isInvalid()); 302 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc), 303 SourceMgr, LangOpts); 304 EXPECT_TRUE(range.isInvalid()); 305 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc), 306 SourceMgr, LangOpts); 307 EXPECT_TRUE(!range.isTokenRange()); 308 EXPECT_EQ(range.getAsRange(), 309 SourceRange(macroRange.getBegin(), 310 macroRange.getEnd().getLocWithOffset(1))); 311 312 StringRef text = Lexer::getSourceText( 313 CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc), 314 SourceMgr, LangOpts); 315 EXPECT_EQ(text, "M(foo)"); 316 317 SourceLocation macroLsqrLoc = toks[3].getLocation(); 318 SourceLocation macroIdLoc = toks[4].getLocation(); 319 SourceLocation macroRsqrLoc = toks[5].getLocation(); 320 SourceLocation fileLsqrLoc = SourceMgr.getSpellingLoc(macroLsqrLoc); 321 SourceLocation fileIdLoc = SourceMgr.getSpellingLoc(macroIdLoc); 322 SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc); 323 324 range = Lexer::makeFileCharRange( 325 CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc), 326 SourceMgr, LangOpts); 327 EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)), 328 range.getAsRange()); 329 330 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc), 331 SourceMgr, LangOpts); 332 EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)), 333 range.getAsRange()); 334 335 macroPair = SourceMgr.getExpansionRange(macroLsqrLoc); 336 range = Lexer::makeFileCharRange( 337 CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc), 338 SourceMgr, LangOpts); 339 EXPECT_EQ(SourceRange(macroPair.first, macroPair.second.getLocWithOffset(1)), 340 range.getAsRange()); 341 342 text = Lexer::getSourceText( 343 CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)), 344 SourceMgr, LangOpts); 345 EXPECT_EQ(text, "[bar"); 346 347 348 SourceLocation idLoc1 = toks[6].getLocation(); 349 SourceLocation idLoc2 = toks[7].getLocation(); 350 SourceLocation idLoc3 = toks[8].getLocation(); 351 SourceLocation idLoc4 = toks[9].getLocation(); 352 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc1, SourceMgr, LangOpts)); 353 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc2, SourceMgr, LangOpts)); 354 EXPECT_EQ("NOF2", Lexer::getImmediateMacroName(idLoc3, SourceMgr, LangOpts)); 355 EXPECT_EQ("N", Lexer::getImmediateMacroName(idLoc4, SourceMgr, LangOpts)); 356 } 357 358 } // anonymous namespace 359
