1 //== HTMLRewrite.cpp - Translate source code into prettified HTML --*- C++ -*-// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file defines the HTMLRewriter clas, which is used to translate the 11 // text of a source file into prettified HTML. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "clang/Rewrite/Core/HTMLRewrite.h" 16 #include "clang/Basic/SourceManager.h" 17 #include "clang/Lex/Preprocessor.h" 18 #include "clang/Lex/TokenConcatenation.h" 19 #include "clang/Rewrite/Core/Rewriter.h" 20 #include "llvm/ADT/OwningPtr.h" 21 #include "llvm/ADT/SmallString.h" 22 #include "llvm/Support/ErrorHandling.h" 23 #include "llvm/Support/MemoryBuffer.h" 24 #include "llvm/Support/raw_ostream.h" 25 using namespace clang; 26 27 28 /// HighlightRange - Highlight a range in the source code with the specified 29 /// start/end tags. B/E must be in the same file. This ensures that 30 /// start/end tags are placed at the start/end of each line if the range is 31 /// multiline. 32 void html::HighlightRange(Rewriter &R, SourceLocation B, SourceLocation E, 33 const char *StartTag, const char *EndTag) { 34 SourceManager &SM = R.getSourceMgr(); 35 B = SM.getExpansionLoc(B); 36 E = SM.getExpansionLoc(E); 37 FileID FID = SM.getFileID(B); 38 assert(SM.getFileID(E) == FID && "B/E not in the same file!"); 39 40 unsigned BOffset = SM.getFileOffset(B); 41 unsigned EOffset = SM.getFileOffset(E); 42 43 // Include the whole end token in the range. 44 EOffset += Lexer::MeasureTokenLength(E, R.getSourceMgr(), R.getLangOpts()); 45 46 bool Invalid = false; 47 const char *BufferStart = SM.getBufferData(FID, &Invalid).data(); 48 if (Invalid) 49 return; 50 51 HighlightRange(R.getEditBuffer(FID), BOffset, EOffset, 52 BufferStart, StartTag, EndTag); 53 } 54 55 /// HighlightRange - This is the same as the above method, but takes 56 /// decomposed file locations. 57 void html::HighlightRange(RewriteBuffer &RB, unsigned B, unsigned E, 58 const char *BufferStart, 59 const char *StartTag, const char *EndTag) { 60 // Insert the tag at the absolute start/end of the range. 61 RB.InsertTextAfter(B, StartTag); 62 RB.InsertTextBefore(E, EndTag); 63 64 // Scan the range to see if there is a \r or \n. If so, and if the line is 65 // not blank, insert tags on that line as well. 66 bool HadOpenTag = true; 67 68 unsigned LastNonWhiteSpace = B; 69 for (unsigned i = B; i != E; ++i) { 70 switch (BufferStart[i]) { 71 case '\r': 72 case '\n': 73 // Okay, we found a newline in the range. If we have an open tag, we need 74 // to insert a close tag at the first non-whitespace before the newline. 75 if (HadOpenTag) 76 RB.InsertTextBefore(LastNonWhiteSpace+1, EndTag); 77 78 // Instead of inserting an open tag immediately after the newline, we 79 // wait until we see a non-whitespace character. This prevents us from 80 // inserting tags around blank lines, and also allows the open tag to 81 // be put *after* whitespace on a non-blank line. 82 HadOpenTag = false; 83 break; 84 case '\0': 85 case ' ': 86 case '\t': 87 case '\f': 88 case '\v': 89 // Ignore whitespace. 90 break; 91 92 default: 93 // If there is no tag open, do it now. 94 if (!HadOpenTag) { 95 RB.InsertTextAfter(i, StartTag); 96 HadOpenTag = true; 97 } 98 99 // Remember this character. 100 LastNonWhiteSpace = i; 101 break; 102 } 103 } 104 } 105 106 void html::EscapeText(Rewriter &R, FileID FID, 107 bool EscapeSpaces, bool ReplaceTabs) { 108 109 const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID); 110 const char* C = Buf->getBufferStart(); 111 const char* FileEnd = Buf->getBufferEnd(); 112 113 assert (C <= FileEnd); 114 115 RewriteBuffer &RB = R.getEditBuffer(FID); 116 117 unsigned ColNo = 0; 118 for (unsigned FilePos = 0; C != FileEnd ; ++C, ++FilePos) { 119 switch (*C) { 120 default: ++ColNo; break; 121 case '\n': 122 case '\r': 123 ColNo = 0; 124 break; 125 126 case ' ': 127 if (EscapeSpaces) 128 RB.ReplaceText(FilePos, 1, " "); 129 ++ColNo; 130 break; 131 case '\f': 132 RB.ReplaceText(FilePos, 1, "<hr>"); 133 ColNo = 0; 134 break; 135 136 case '\t': { 137 if (!ReplaceTabs) 138 break; 139 unsigned NumSpaces = 8-(ColNo&7); 140 if (EscapeSpaces) 141 RB.ReplaceText(FilePos, 1, 142 StringRef(" " 143 " ", 6*NumSpaces)); 144 else 145 RB.ReplaceText(FilePos, 1, StringRef(" ", NumSpaces)); 146 ColNo += NumSpaces; 147 break; 148 } 149 case '<': 150 RB.ReplaceText(FilePos, 1, "<"); 151 ++ColNo; 152 break; 153 154 case '>': 155 RB.ReplaceText(FilePos, 1, ">"); 156 ++ColNo; 157 break; 158 159 case '&': 160 RB.ReplaceText(FilePos, 1, "&"); 161 ++ColNo; 162 break; 163 } 164 } 165 } 166 167 std::string html::EscapeText(const std::string& s, bool EscapeSpaces, 168 bool ReplaceTabs) { 169 170 unsigned len = s.size(); 171 std::string Str; 172 llvm::raw_string_ostream os(Str); 173 174 for (unsigned i = 0 ; i < len; ++i) { 175 176 char c = s[i]; 177 switch (c) { 178 default: 179 os << c; break; 180 181 case ' ': 182 if (EscapeSpaces) os << " "; 183 else os << ' '; 184 break; 185 186 case '\t': 187 if (ReplaceTabs) { 188 if (EscapeSpaces) 189 for (unsigned i = 0; i < 4; ++i) 190 os << " "; 191 else 192 for (unsigned i = 0; i < 4; ++i) 193 os << " "; 194 } 195 else 196 os << c; 197 198 break; 199 200 case '<': os << "<"; break; 201 case '>': os << ">"; break; 202 case '&': os << "&"; break; 203 } 204 } 205 206 return os.str(); 207 } 208 209 static void AddLineNumber(RewriteBuffer &RB, unsigned LineNo, 210 unsigned B, unsigned E) { 211 SmallString<256> Str; 212 llvm::raw_svector_ostream OS(Str); 213 214 OS << "<tr><td class=\"num\" id=\"LN" 215 << LineNo << "\">" 216 << LineNo << "</td><td class=\"line\">"; 217 218 if (B == E) { // Handle empty lines. 219 OS << " </td></tr>"; 220 RB.InsertTextBefore(B, OS.str()); 221 } else { 222 RB.InsertTextBefore(B, OS.str()); 223 RB.InsertTextBefore(E, "</td></tr>"); 224 } 225 } 226 227 void html::AddLineNumbers(Rewriter& R, FileID FID) { 228 229 const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID); 230 const char* FileBeg = Buf->getBufferStart(); 231 const char* FileEnd = Buf->getBufferEnd(); 232 const char* C = FileBeg; 233 RewriteBuffer &RB = R.getEditBuffer(FID); 234 235 assert (C <= FileEnd); 236 237 unsigned LineNo = 0; 238 unsigned FilePos = 0; 239 240 while (C != FileEnd) { 241 242 ++LineNo; 243 unsigned LineStartPos = FilePos; 244 unsigned LineEndPos = FileEnd - FileBeg; 245 246 assert (FilePos <= LineEndPos); 247 assert (C < FileEnd); 248 249 // Scan until the newline (or end-of-file). 250 251 while (C != FileEnd) { 252 char c = *C; 253 ++C; 254 255 if (c == '\n') { 256 LineEndPos = FilePos++; 257 break; 258 } 259 260 ++FilePos; 261 } 262 263 AddLineNumber(RB, LineNo, LineStartPos, LineEndPos); 264 } 265 266 // Add one big table tag that surrounds all of the code. 267 RB.InsertTextBefore(0, "<table class=\"code\">\n"); 268 RB.InsertTextAfter(FileEnd - FileBeg, "</table>"); 269 } 270 271 void html::AddHeaderFooterInternalBuiltinCSS(Rewriter& R, FileID FID, 272 const char *title) { 273 274 const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID); 275 const char* FileStart = Buf->getBufferStart(); 276 const char* FileEnd = Buf->getBufferEnd(); 277 278 SourceLocation StartLoc = R.getSourceMgr().getLocForStartOfFile(FID); 279 SourceLocation EndLoc = StartLoc.getLocWithOffset(FileEnd-FileStart); 280 281 std::string s; 282 llvm::raw_string_ostream os(s); 283 os << "<!doctype html>\n" // Use HTML 5 doctype 284 "<html>\n<head>\n"; 285 286 if (title) 287 os << "<title>" << html::EscapeText(title) << "</title>\n"; 288 289 os << "<style type=\"text/css\">\n" 290 " body { color:#000000; background-color:#ffffff }\n" 291 " body { font-family:Helvetica, sans-serif; font-size:10pt }\n" 292 " h1 { font-size:14pt }\n" 293 " .code { border-collapse:collapse; width:100%; }\n" 294 " .code { font-family: \"Monospace\", monospace; font-size:10pt }\n" 295 " .code { line-height: 1.2em }\n" 296 " .comment { color: green; font-style: oblique }\n" 297 " .keyword { color: blue }\n" 298 " .string_literal { color: red }\n" 299 " .directive { color: darkmagenta }\n" 300 // Macro expansions. 301 " .expansion { display: none; }\n" 302 " .macro:hover .expansion { display: block; border: 2px solid #FF0000; " 303 "padding: 2px; background-color:#FFF0F0; font-weight: normal; " 304 " -webkit-border-radius:5px; -webkit-box-shadow:1px 1px 7px #000; " 305 "position: absolute; top: -1em; left:10em; z-index: 1 } \n" 306 " .macro { color: darkmagenta; background-color:LemonChiffon;" 307 // Macros are position: relative to provide base for expansions. 308 " position: relative }\n" 309 " .num { width:2.5em; padding-right:2ex; background-color:#eeeeee }\n" 310 " .num { text-align:right; font-size:8pt }\n" 311 " .num { color:#444444 }\n" 312 " .line { padding-left: 1ex; border-left: 3px solid #ccc }\n" 313 " .line { white-space: pre }\n" 314 " .msg { -webkit-box-shadow:1px 1px 7px #000 }\n" 315 " .msg { -webkit-border-radius:5px }\n" 316 " .msg { font-family:Helvetica, sans-serif; font-size:8pt }\n" 317 " .msg { float:left }\n" 318 " .msg { padding:0.25em 1ex 0.25em 1ex }\n" 319 " .msg { margin-top:10px; margin-bottom:10px }\n" 320 " .msg { font-weight:bold }\n" 321 " .msg { max-width:60em; word-wrap: break-word; white-space: pre-wrap }\n" 322 " .msgT { padding:0x; spacing:0x }\n" 323 " .msgEvent { background-color:#fff8b4; color:#000000 }\n" 324 " .msgControl { background-color:#bbbbbb; color:#000000 }\n" 325 " .mrange { background-color:#dfddf3 }\n" 326 " .mrange { border-bottom:1px solid #6F9DBE }\n" 327 " .PathIndex { font-weight: bold; padding:0px 5px; " 328 "margin-right:5px; }\n" 329 " .PathIndex { -webkit-border-radius:8px }\n" 330 " .PathIndexEvent { background-color:#bfba87 }\n" 331 " .PathIndexControl { background-color:#8c8c8c }\n" 332 " .PathNav a { text-decoration:none; font-size: larger }\n" 333 " .CodeInsertionHint { font-weight: bold; background-color: #10dd10 }\n" 334 " .CodeRemovalHint { background-color:#de1010 }\n" 335 " .CodeRemovalHint { border-bottom:1px solid #6F9DBE }\n" 336 " table.simpletable {\n" 337 " padding: 5px;\n" 338 " font-size:12pt;\n" 339 " margin:20px;\n" 340 " border-collapse: collapse; border-spacing: 0px;\n" 341 " }\n" 342 " td.rowname {\n" 343 " text-align:right; font-weight:bold; color:#444444;\n" 344 " padding-right:2ex; }\n" 345 "</style>\n</head>\n<body>"; 346 347 // Generate header 348 R.InsertTextBefore(StartLoc, os.str()); 349 // Generate footer 350 351 R.InsertTextAfter(EndLoc, "</body></html>\n"); 352 } 353 354 /// SyntaxHighlight - Relex the specified FileID and annotate the HTML with 355 /// information about keywords, macro expansions etc. This uses the macro 356 /// table state from the end of the file, so it won't be perfectly perfect, 357 /// but it will be reasonably close. 358 void html::SyntaxHighlight(Rewriter &R, FileID FID, const Preprocessor &PP) { 359 RewriteBuffer &RB = R.getEditBuffer(FID); 360 361 const SourceManager &SM = PP.getSourceManager(); 362 const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID); 363 Lexer L(FID, FromFile, SM, PP.getLangOpts()); 364 const char *BufferStart = L.getBufferStart(); 365 366 // Inform the preprocessor that we want to retain comments as tokens, so we 367 // can highlight them. 368 L.SetCommentRetentionState(true); 369 370 // Lex all the tokens in raw mode, to avoid entering #includes or expanding 371 // macros. 372 Token Tok; 373 L.LexFromRawLexer(Tok); 374 375 while (Tok.isNot(tok::eof)) { 376 // Since we are lexing unexpanded tokens, all tokens are from the main 377 // FileID. 378 unsigned TokOffs = SM.getFileOffset(Tok.getLocation()); 379 unsigned TokLen = Tok.getLength(); 380 switch (Tok.getKind()) { 381 default: break; 382 case tok::identifier: 383 llvm_unreachable("tok::identifier in raw lexing mode!"); 384 case tok::raw_identifier: { 385 // Fill in Result.IdentifierInfo and update the token kind, 386 // looking up the identifier in the identifier table. 387 PP.LookUpIdentifierInfo(Tok); 388 389 // If this is a pp-identifier, for a keyword, highlight it as such. 390 if (Tok.isNot(tok::identifier)) 391 HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart, 392 "<span class='keyword'>", "</span>"); 393 break; 394 } 395 case tok::comment: 396 HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart, 397 "<span class='comment'>", "</span>"); 398 break; 399 case tok::utf8_string_literal: 400 // Chop off the u part of u8 prefix 401 ++TokOffs; 402 --TokLen; 403 // FALL THROUGH to chop the 8 404 case tok::wide_string_literal: 405 case tok::utf16_string_literal: 406 case tok::utf32_string_literal: 407 // Chop off the L, u, U or 8 prefix 408 ++TokOffs; 409 --TokLen; 410 // FALL THROUGH. 411 case tok::string_literal: 412 // FIXME: Exclude the optional ud-suffix from the highlighted range. 413 HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart, 414 "<span class='string_literal'>", "</span>"); 415 break; 416 case tok::hash: { 417 // If this is a preprocessor directive, all tokens to end of line are too. 418 if (!Tok.isAtStartOfLine()) 419 break; 420 421 // Eat all of the tokens until we get to the next one at the start of 422 // line. 423 unsigned TokEnd = TokOffs+TokLen; 424 L.LexFromRawLexer(Tok); 425 while (!Tok.isAtStartOfLine() && Tok.isNot(tok::eof)) { 426 TokEnd = SM.getFileOffset(Tok.getLocation())+Tok.getLength(); 427 L.LexFromRawLexer(Tok); 428 } 429 430 // Find end of line. This is a hack. 431 HighlightRange(RB, TokOffs, TokEnd, BufferStart, 432 "<span class='directive'>", "</span>"); 433 434 // Don't skip the next token. 435 continue; 436 } 437 } 438 439 L.LexFromRawLexer(Tok); 440 } 441 } 442 443 /// HighlightMacros - This uses the macro table state from the end of the 444 /// file, to re-expand macros and insert (into the HTML) information about the 445 /// macro expansions. This won't be perfectly perfect, but it will be 446 /// reasonably close. 447 void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) { 448 // Re-lex the raw token stream into a token buffer. 449 const SourceManager &SM = PP.getSourceManager(); 450 std::vector<Token> TokenStream; 451 452 const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID); 453 Lexer L(FID, FromFile, SM, PP.getLangOpts()); 454 455 // Lex all the tokens in raw mode, to avoid entering #includes or expanding 456 // macros. 457 while (1) { 458 Token Tok; 459 L.LexFromRawLexer(Tok); 460 461 // If this is a # at the start of a line, discard it from the token stream. 462 // We don't want the re-preprocess step to see #defines, #includes or other 463 // preprocessor directives. 464 if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) 465 continue; 466 467 // If this is a ## token, change its kind to unknown so that repreprocessing 468 // it will not produce an error. 469 if (Tok.is(tok::hashhash)) 470 Tok.setKind(tok::unknown); 471 472 // If this raw token is an identifier, the raw lexer won't have looked up 473 // the corresponding identifier info for it. Do this now so that it will be 474 // macro expanded when we re-preprocess it. 475 if (Tok.is(tok::raw_identifier)) 476 PP.LookUpIdentifierInfo(Tok); 477 478 TokenStream.push_back(Tok); 479 480 if (Tok.is(tok::eof)) break; 481 } 482 483 // Temporarily change the diagnostics object so that we ignore any generated 484 // diagnostics from this pass. 485 DiagnosticsEngine TmpDiags(PP.getDiagnostics().getDiagnosticIDs(), 486 &PP.getDiagnostics().getDiagnosticOptions(), 487 new IgnoringDiagConsumer); 488 489 // FIXME: This is a huge hack; we reuse the input preprocessor because we want 490 // its state, but we aren't actually changing it (we hope). This should really 491 // construct a copy of the preprocessor. 492 Preprocessor &TmpPP = const_cast<Preprocessor&>(PP); 493 DiagnosticsEngine *OldDiags = &TmpPP.getDiagnostics(); 494 TmpPP.setDiagnostics(TmpDiags); 495 496 // Inform the preprocessor that we don't want comments. 497 TmpPP.SetCommentRetentionState(false, false); 498 499 // We don't want pragmas either. Although we filtered out #pragma, removing 500 // _Pragma and __pragma is much harder. 501 bool PragmasPreviouslyEnabled = TmpPP.getPragmasEnabled(); 502 TmpPP.setPragmasEnabled(false); 503 504 // Enter the tokens we just lexed. This will cause them to be macro expanded 505 // but won't enter sub-files (because we removed #'s). 506 TmpPP.EnterTokenStream(&TokenStream[0], TokenStream.size(), false, false); 507 508 TokenConcatenation ConcatInfo(TmpPP); 509 510 // Lex all the tokens. 511 Token Tok; 512 TmpPP.Lex(Tok); 513 while (Tok.isNot(tok::eof)) { 514 // Ignore non-macro tokens. 515 if (!Tok.getLocation().isMacroID()) { 516 TmpPP.Lex(Tok); 517 continue; 518 } 519 520 // Okay, we have the first token of a macro expansion: highlight the 521 // expansion by inserting a start tag before the macro expansion and 522 // end tag after it. 523 std::pair<SourceLocation, SourceLocation> LLoc = 524 SM.getExpansionRange(Tok.getLocation()); 525 526 // Ignore tokens whose instantiation location was not the main file. 527 if (SM.getFileID(LLoc.first) != FID) { 528 TmpPP.Lex(Tok); 529 continue; 530 } 531 532 assert(SM.getFileID(LLoc.second) == FID && 533 "Start and end of expansion must be in the same ultimate file!"); 534 535 std::string Expansion = EscapeText(TmpPP.getSpelling(Tok)); 536 unsigned LineLen = Expansion.size(); 537 538 Token PrevPrevTok; 539 Token PrevTok = Tok; 540 // Okay, eat this token, getting the next one. 541 TmpPP.Lex(Tok); 542 543 // Skip all the rest of the tokens that are part of this macro 544 // instantiation. It would be really nice to pop up a window with all the 545 // spelling of the tokens or something. 546 while (!Tok.is(tok::eof) && 547 SM.getExpansionLoc(Tok.getLocation()) == LLoc.first) { 548 // Insert a newline if the macro expansion is getting large. 549 if (LineLen > 60) { 550 Expansion += "<br>"; 551 LineLen = 0; 552 } 553 554 LineLen -= Expansion.size(); 555 556 // If the tokens were already space separated, or if they must be to avoid 557 // them being implicitly pasted, add a space between them. 558 if (Tok.hasLeadingSpace() || 559 ConcatInfo.AvoidConcat(PrevPrevTok, PrevTok, Tok)) 560 Expansion += ' '; 561 562 // Escape any special characters in the token text. 563 Expansion += EscapeText(TmpPP.getSpelling(Tok)); 564 LineLen += Expansion.size(); 565 566 PrevPrevTok = PrevTok; 567 PrevTok = Tok; 568 TmpPP.Lex(Tok); 569 } 570 571 572 // Insert the expansion as the end tag, so that multi-line macros all get 573 // highlighted. 574 Expansion = "<span class='expansion'>" + Expansion + "</span></span>"; 575 576 HighlightRange(R, LLoc.first, LLoc.second, 577 "<span class='macro'>", Expansion.c_str()); 578 } 579 580 // Restore the preprocessor's old state. 581 TmpPP.setDiagnostics(*OldDiags); 582 TmpPP.setPragmasEnabled(PragmasPreviouslyEnabled); 583 } 584