Home | History | Annotate | Download | only in Core
      1 //== HTMLRewrite.cpp - Translate source code into prettified HTML --*- C++ -*-//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 //  This file defines the HTMLRewriter class, which is used to translate the
     11 //  text of a source file into prettified HTML.
     12 //
     13 //===----------------------------------------------------------------------===//
     14 
     15 #include "clang/Rewrite/Core/HTMLRewrite.h"
     16 #include "clang/Basic/SourceManager.h"
     17 #include "clang/Lex/Preprocessor.h"
     18 #include "clang/Lex/TokenConcatenation.h"
     19 #include "clang/Rewrite/Core/Rewriter.h"
     20 #include "llvm/ADT/SmallString.h"
     21 #include "llvm/Support/ErrorHandling.h"
     22 #include "llvm/Support/MemoryBuffer.h"
     23 #include "llvm/Support/raw_ostream.h"
     24 #include <memory>
     25 using namespace clang;
     26 
     27 
     28 /// HighlightRange - Highlight a range in the source code with the specified
     29 /// start/end tags.  B/E must be in the same file.  This ensures that
     30 /// start/end tags are placed at the start/end of each line if the range is
     31 /// multiline.
     32 void html::HighlightRange(Rewriter &R, SourceLocation B, SourceLocation E,
     33                           const char *StartTag, const char *EndTag) {
     34   SourceManager &SM = R.getSourceMgr();
     35   B = SM.getExpansionLoc(B);
     36   E = SM.getExpansionLoc(E);
     37   FileID FID = SM.getFileID(B);
     38   assert(SM.getFileID(E) == FID && "B/E not in the same file!");
     39 
     40   unsigned BOffset = SM.getFileOffset(B);
     41   unsigned EOffset = SM.getFileOffset(E);
     42 
     43   // Include the whole end token in the range.
     44   EOffset += Lexer::MeasureTokenLength(E, R.getSourceMgr(), R.getLangOpts());
     45 
     46   bool Invalid = false;
     47   const char *BufferStart = SM.getBufferData(FID, &Invalid).data();
     48   if (Invalid)
     49     return;
     50 
     51   HighlightRange(R.getEditBuffer(FID), BOffset, EOffset,
     52                  BufferStart, StartTag, EndTag);
     53 }
     54 
     55 /// HighlightRange - This is the same as the above method, but takes
     56 /// decomposed file locations.
     57 void html::HighlightRange(RewriteBuffer &RB, unsigned B, unsigned E,
     58                           const char *BufferStart,
     59                           const char *StartTag, const char *EndTag) {
     60   // Insert the tag at the absolute start/end of the range.
     61   RB.InsertTextAfter(B, StartTag);
     62   RB.InsertTextBefore(E, EndTag);
     63 
     64   // Scan the range to see if there is a \r or \n.  If so, and if the line is
     65   // not blank, insert tags on that line as well.
     66   bool HadOpenTag = true;
     67 
     68   unsigned LastNonWhiteSpace = B;
     69   for (unsigned i = B; i != E; ++i) {
     70     switch (BufferStart[i]) {
     71     case '\r':
     72     case '\n':
     73       // Okay, we found a newline in the range.  If we have an open tag, we need
     74       // to insert a close tag at the first non-whitespace before the newline.
     75       if (HadOpenTag)
     76         RB.InsertTextBefore(LastNonWhiteSpace+1, EndTag);
     77 
     78       // Instead of inserting an open tag immediately after the newline, we
     79       // wait until we see a non-whitespace character.  This prevents us from
     80       // inserting tags around blank lines, and also allows the open tag to
     81       // be put *after* whitespace on a non-blank line.
     82       HadOpenTag = false;
     83       break;
     84     case '\0':
     85     case ' ':
     86     case '\t':
     87     case '\f':
     88     case '\v':
     89       // Ignore whitespace.
     90       break;
     91 
     92     default:
     93       // If there is no tag open, do it now.
     94       if (!HadOpenTag) {
     95         RB.InsertTextAfter(i, StartTag);
     96         HadOpenTag = true;
     97       }
     98 
     99       // Remember this character.
    100       LastNonWhiteSpace = i;
    101       break;
    102     }
    103   }
    104 }
    105 
    106 void html::EscapeText(Rewriter &R, FileID FID,
    107                       bool EscapeSpaces, bool ReplaceTabs) {
    108 
    109   const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID);
    110   const char* C = Buf->getBufferStart();
    111   const char* FileEnd = Buf->getBufferEnd();
    112 
    113   assert (C <= FileEnd);
    114 
    115   RewriteBuffer &RB = R.getEditBuffer(FID);
    116 
    117   unsigned ColNo = 0;
    118   for (unsigned FilePos = 0; C != FileEnd ; ++C, ++FilePos) {
    119     switch (*C) {
    120     default: ++ColNo; break;
    121     case '\n':
    122     case '\r':
    123       ColNo = 0;
    124       break;
    125 
    126     case ' ':
    127       if (EscapeSpaces)
    128         RB.ReplaceText(FilePos, 1, "&nbsp;");
    129       ++ColNo;
    130       break;
    131     case '\f':
    132       RB.ReplaceText(FilePos, 1, "<hr>");
    133       ColNo = 0;
    134       break;
    135 
    136     case '\t': {
    137       if (!ReplaceTabs)
    138         break;
    139       unsigned NumSpaces = 8-(ColNo&7);
    140       if (EscapeSpaces)
    141         RB.ReplaceText(FilePos, 1,
    142                        StringRef("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;"
    143                                        "&nbsp;&nbsp;&nbsp;", 6*NumSpaces));
    144       else
    145         RB.ReplaceText(FilePos, 1, StringRef("        ", NumSpaces));
    146       ColNo += NumSpaces;
    147       break;
    148     }
    149     case '<':
    150       RB.ReplaceText(FilePos, 1, "&lt;");
    151       ++ColNo;
    152       break;
    153 
    154     case '>':
    155       RB.ReplaceText(FilePos, 1, "&gt;");
    156       ++ColNo;
    157       break;
    158 
    159     case '&':
    160       RB.ReplaceText(FilePos, 1, "&amp;");
    161       ++ColNo;
    162       break;
    163     }
    164   }
    165 }
    166 
    167 std::string html::EscapeText(StringRef s, bool EscapeSpaces, bool ReplaceTabs) {
    168 
    169   unsigned len = s.size();
    170   std::string Str;
    171   llvm::raw_string_ostream os(Str);
    172 
    173   for (unsigned i = 0 ; i < len; ++i) {
    174 
    175     char c = s[i];
    176     switch (c) {
    177     default:
    178       os << c; break;
    179 
    180     case ' ':
    181       if (EscapeSpaces) os << "&nbsp;";
    182       else os << ' ';
    183       break;
    184 
    185     case '\t':
    186       if (ReplaceTabs) {
    187         if (EscapeSpaces)
    188           for (unsigned i = 0; i < 4; ++i)
    189             os << "&nbsp;";
    190         else
    191           for (unsigned i = 0; i < 4; ++i)
    192             os << " ";
    193       }
    194       else
    195         os << c;
    196 
    197       break;
    198 
    199     case '<': os << "&lt;"; break;
    200     case '>': os << "&gt;"; break;
    201     case '&': os << "&amp;"; break;
    202     }
    203   }
    204 
    205   return os.str();
    206 }
    207 
    208 static void AddLineNumber(RewriteBuffer &RB, unsigned LineNo,
    209                           unsigned B, unsigned E) {
    210   SmallString<256> Str;
    211   llvm::raw_svector_ostream OS(Str);
    212 
    213   OS << "<tr><td class=\"num\" id=\"LN"
    214      << LineNo << "\">"
    215      << LineNo << "</td><td class=\"line\">";
    216 
    217   if (B == E) { // Handle empty lines.
    218     OS << " </td></tr>";
    219     RB.InsertTextBefore(B, OS.str());
    220   } else {
    221     RB.InsertTextBefore(B, OS.str());
    222     RB.InsertTextBefore(E, "</td></tr>");
    223   }
    224 }
    225 
    226 void html::AddLineNumbers(Rewriter& R, FileID FID) {
    227 
    228   const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID);
    229   const char* FileBeg = Buf->getBufferStart();
    230   const char* FileEnd = Buf->getBufferEnd();
    231   const char* C = FileBeg;
    232   RewriteBuffer &RB = R.getEditBuffer(FID);
    233 
    234   assert (C <= FileEnd);
    235 
    236   unsigned LineNo = 0;
    237   unsigned FilePos = 0;
    238 
    239   while (C != FileEnd) {
    240 
    241     ++LineNo;
    242     unsigned LineStartPos = FilePos;
    243     unsigned LineEndPos = FileEnd - FileBeg;
    244 
    245     assert (FilePos <= LineEndPos);
    246     assert (C < FileEnd);
    247 
    248     // Scan until the newline (or end-of-file).
    249 
    250     while (C != FileEnd) {
    251       char c = *C;
    252       ++C;
    253 
    254       if (c == '\n') {
    255         LineEndPos = FilePos++;
    256         break;
    257       }
    258 
    259       ++FilePos;
    260     }
    261 
    262     AddLineNumber(RB, LineNo, LineStartPos, LineEndPos);
    263   }
    264 
    265   // Add one big table tag that surrounds all of the code.
    266   RB.InsertTextBefore(0, "<table class=\"code\">\n");
    267   RB.InsertTextAfter(FileEnd - FileBeg, "</table>");
    268 }
    269 
    270 void html::AddHeaderFooterInternalBuiltinCSS(Rewriter& R, FileID FID,
    271                                              const char *title) {
    272 
    273   const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID);
    274   const char* FileStart = Buf->getBufferStart();
    275   const char* FileEnd = Buf->getBufferEnd();
    276 
    277   SourceLocation StartLoc = R.getSourceMgr().getLocForStartOfFile(FID);
    278   SourceLocation EndLoc = StartLoc.getLocWithOffset(FileEnd-FileStart);
    279 
    280   std::string s;
    281   llvm::raw_string_ostream os(s);
    282   os << "<!doctype html>\n" // Use HTML 5 doctype
    283         "<html>\n<head>\n";
    284 
    285   if (title)
    286     os << "<title>" << html::EscapeText(title) << "</title>\n";
    287 
    288   os << "<style type=\"text/css\">\n"
    289       " body { color:#000000; background-color:#ffffff }\n"
    290       " body { font-family:Helvetica, sans-serif; font-size:10pt }\n"
    291       " h1 { font-size:14pt }\n"
    292       " .code { border-collapse:collapse; width:100%; }\n"
    293       " .code { font-family: \"Monospace\", monospace; font-size:10pt }\n"
    294       " .code { line-height: 1.2em }\n"
    295       " .comment { color: green; font-style: oblique }\n"
    296       " .keyword { color: blue }\n"
    297       " .string_literal { color: red }\n"
    298       " .directive { color: darkmagenta }\n"
    299       // Macro expansions.
    300       " .expansion { display: none; }\n"
    301       " .macro:hover .expansion { display: block; border: 2px solid #FF0000; "
    302           "padding: 2px; background-color:#FFF0F0; font-weight: normal; "
    303           "  -webkit-border-radius:5px;  -webkit-box-shadow:1px 1px 7px #000; "
    304           "position: absolute; top: -1em; left:10em; z-index: 1 } \n"
    305       " .macro { color: darkmagenta; background-color:LemonChiffon;"
    306              // Macros are position: relative to provide base for expansions.
    307              " position: relative }\n"
    308       " .num { width:2.5em; padding-right:2ex; background-color:#eeeeee }\n"
    309       " .num { text-align:right; font-size:8pt }\n"
    310       " .num { color:#444444 }\n"
    311       " .line { padding-left: 1ex; border-left: 3px solid #ccc }\n"
    312       " .line { white-space: pre }\n"
    313       " .msg { -webkit-box-shadow:1px 1px 7px #000 }\n"
    314       " .msg { -webkit-border-radius:5px }\n"
    315       " .msg { font-family:Helvetica, sans-serif; font-size:8pt }\n"
    316       " .msg { float:left }\n"
    317       " .msg { padding:0.25em 1ex 0.25em 1ex }\n"
    318       " .msg { margin-top:10px; margin-bottom:10px }\n"
    319       " .msg { font-weight:bold }\n"
    320       " .msg { max-width:60em; word-wrap: break-word; white-space: pre-wrap }\n"
    321       " .msgT { padding:0x; spacing:0x }\n"
    322       " .msgEvent { background-color:#fff8b4; color:#000000 }\n"
    323       " .msgControl { background-color:#bbbbbb; color:#000000 }\n"
    324       " .mrange { background-color:#dfddf3 }\n"
    325       " .mrange { border-bottom:1px solid #6F9DBE }\n"
    326       " .PathIndex { font-weight: bold; padding:0px 5px; "
    327         "margin-right:5px; }\n"
    328       " .PathIndex { -webkit-border-radius:8px }\n"
    329       " .PathIndexEvent { background-color:#bfba87 }\n"
    330       " .PathIndexControl { background-color:#8c8c8c }\n"
    331       " .PathNav a { text-decoration:none; font-size: larger }\n"
    332       " .CodeInsertionHint { font-weight: bold; background-color: #10dd10 }\n"
    333       " .CodeRemovalHint { background-color:#de1010 }\n"
    334       " .CodeRemovalHint { border-bottom:1px solid #6F9DBE }\n"
    335       " table.simpletable {\n"
    336       "   padding: 5px;\n"
    337       "   font-size:12pt;\n"
    338       "   margin:20px;\n"
    339       "   border-collapse: collapse; border-spacing: 0px;\n"
    340       " }\n"
    341       " td.rowname {\n"
    342       "   text-align:right; font-weight:bold; color:#444444;\n"
    343       "   padding-right:2ex; }\n"
    344       "</style>\n</head>\n<body>";
    345 
    346   // Generate header
    347   R.InsertTextBefore(StartLoc, os.str());
    348   // Generate footer
    349 
    350   R.InsertTextAfter(EndLoc, "</body></html>\n");
    351 }
    352 
    353 /// SyntaxHighlight - Relex the specified FileID and annotate the HTML with
    354 /// information about keywords, macro expansions etc.  This uses the macro
    355 /// table state from the end of the file, so it won't be perfectly perfect,
    356 /// but it will be reasonably close.
    357 void html::SyntaxHighlight(Rewriter &R, FileID FID, const Preprocessor &PP) {
    358   RewriteBuffer &RB = R.getEditBuffer(FID);
    359 
    360   const SourceManager &SM = PP.getSourceManager();
    361   const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
    362   Lexer L(FID, FromFile, SM, PP.getLangOpts());
    363   const char *BufferStart = L.getBuffer().data();
    364 
    365   // Inform the preprocessor that we want to retain comments as tokens, so we
    366   // can highlight them.
    367   L.SetCommentRetentionState(true);
    368 
    369   // Lex all the tokens in raw mode, to avoid entering #includes or expanding
    370   // macros.
    371   Token Tok;
    372   L.LexFromRawLexer(Tok);
    373 
    374   while (Tok.isNot(tok::eof)) {
    375     // Since we are lexing unexpanded tokens, all tokens are from the main
    376     // FileID.
    377     unsigned TokOffs = SM.getFileOffset(Tok.getLocation());
    378     unsigned TokLen = Tok.getLength();
    379     switch (Tok.getKind()) {
    380     default: break;
    381     case tok::identifier:
    382       llvm_unreachable("tok::identifier in raw lexing mode!");
    383     case tok::raw_identifier: {
    384       // Fill in Result.IdentifierInfo and update the token kind,
    385       // looking up the identifier in the identifier table.
    386       PP.LookUpIdentifierInfo(Tok);
    387 
    388       // If this is a pp-identifier, for a keyword, highlight it as such.
    389       if (Tok.isNot(tok::identifier))
    390         HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
    391                        "<span class='keyword'>", "</span>");
    392       break;
    393     }
    394     case tok::comment:
    395       HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
    396                      "<span class='comment'>", "</span>");
    397       break;
    398     case tok::utf8_string_literal:
    399       // Chop off the u part of u8 prefix
    400       ++TokOffs;
    401       --TokLen;
    402       // FALL THROUGH to chop the 8
    403     case tok::wide_string_literal:
    404     case tok::utf16_string_literal:
    405     case tok::utf32_string_literal:
    406       // Chop off the L, u, U or 8 prefix
    407       ++TokOffs;
    408       --TokLen;
    409       // FALL THROUGH.
    410     case tok::string_literal:
    411       // FIXME: Exclude the optional ud-suffix from the highlighted range.
    412       HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
    413                      "<span class='string_literal'>", "</span>");
    414       break;
    415     case tok::hash: {
    416       // If this is a preprocessor directive, all tokens to end of line are too.
    417       if (!Tok.isAtStartOfLine())
    418         break;
    419 
    420       // Eat all of the tokens until we get to the next one at the start of
    421       // line.
    422       unsigned TokEnd = TokOffs+TokLen;
    423       L.LexFromRawLexer(Tok);
    424       while (!Tok.isAtStartOfLine() && Tok.isNot(tok::eof)) {
    425         TokEnd = SM.getFileOffset(Tok.getLocation())+Tok.getLength();
    426         L.LexFromRawLexer(Tok);
    427       }
    428 
    429       // Find end of line.  This is a hack.
    430       HighlightRange(RB, TokOffs, TokEnd, BufferStart,
    431                      "<span class='directive'>", "</span>");
    432 
    433       // Don't skip the next token.
    434       continue;
    435     }
    436     }
    437 
    438     L.LexFromRawLexer(Tok);
    439   }
    440 }
    441 
    442 /// HighlightMacros - This uses the macro table state from the end of the
    443 /// file, to re-expand macros and insert (into the HTML) information about the
    444 /// macro expansions.  This won't be perfectly perfect, but it will be
    445 /// reasonably close.
    446 void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
    447   // Re-lex the raw token stream into a token buffer.
    448   const SourceManager &SM = PP.getSourceManager();
    449   std::vector<Token> TokenStream;
    450 
    451   const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
    452   Lexer L(FID, FromFile, SM, PP.getLangOpts());
    453 
    454   // Lex all the tokens in raw mode, to avoid entering #includes or expanding
    455   // macros.
    456   while (1) {
    457     Token Tok;
    458     L.LexFromRawLexer(Tok);
    459 
    460     // If this is a # at the start of a line, discard it from the token stream.
    461     // We don't want the re-preprocess step to see #defines, #includes or other
    462     // preprocessor directives.
    463     if (Tok.is(tok::hash) && Tok.isAtStartOfLine())
    464       continue;
    465 
    466     // If this is a ## token, change its kind to unknown so that repreprocessing
    467     // it will not produce an error.
    468     if (Tok.is(tok::hashhash))
    469       Tok.setKind(tok::unknown);
    470 
    471     // If this raw token is an identifier, the raw lexer won't have looked up
    472     // the corresponding identifier info for it.  Do this now so that it will be
    473     // macro expanded when we re-preprocess it.
    474     if (Tok.is(tok::raw_identifier))
    475       PP.LookUpIdentifierInfo(Tok);
    476 
    477     TokenStream.push_back(Tok);
    478 
    479     if (Tok.is(tok::eof)) break;
    480   }
    481 
    482   // Temporarily change the diagnostics object so that we ignore any generated
    483   // diagnostics from this pass.
    484   DiagnosticsEngine TmpDiags(PP.getDiagnostics().getDiagnosticIDs(),
    485                              &PP.getDiagnostics().getDiagnosticOptions(),
    486                       new IgnoringDiagConsumer);
    487 
    488   // FIXME: This is a huge hack; we reuse the input preprocessor because we want
    489   // its state, but we aren't actually changing it (we hope). This should really
    490   // construct a copy of the preprocessor.
    491   Preprocessor &TmpPP = const_cast<Preprocessor&>(PP);
    492   DiagnosticsEngine *OldDiags = &TmpPP.getDiagnostics();
    493   TmpPP.setDiagnostics(TmpDiags);
    494 
    495   // Inform the preprocessor that we don't want comments.
    496   TmpPP.SetCommentRetentionState(false, false);
    497 
    498   // We don't want pragmas either. Although we filtered out #pragma, removing
    499   // _Pragma and __pragma is much harder.
    500   bool PragmasPreviouslyEnabled = TmpPP.getPragmasEnabled();
    501   TmpPP.setPragmasEnabled(false);
    502 
    503   // Enter the tokens we just lexed.  This will cause them to be macro expanded
    504   // but won't enter sub-files (because we removed #'s).
    505   TmpPP.EnterTokenStream(&TokenStream[0], TokenStream.size(), false, false);
    506 
    507   TokenConcatenation ConcatInfo(TmpPP);
    508 
    509   // Lex all the tokens.
    510   Token Tok;
    511   TmpPP.Lex(Tok);
    512   while (Tok.isNot(tok::eof)) {
    513     // Ignore non-macro tokens.
    514     if (!Tok.getLocation().isMacroID()) {
    515       TmpPP.Lex(Tok);
    516       continue;
    517     }
    518 
    519     // Okay, we have the first token of a macro expansion: highlight the
    520     // expansion by inserting a start tag before the macro expansion and
    521     // end tag after it.
    522     std::pair<SourceLocation, SourceLocation> LLoc =
    523       SM.getExpansionRange(Tok.getLocation());
    524 
    525     // Ignore tokens whose instantiation location was not the main file.
    526     if (SM.getFileID(LLoc.first) != FID) {
    527       TmpPP.Lex(Tok);
    528       continue;
    529     }
    530 
    531     assert(SM.getFileID(LLoc.second) == FID &&
    532            "Start and end of expansion must be in the same ultimate file!");
    533 
    534     std::string Expansion = EscapeText(TmpPP.getSpelling(Tok));
    535     unsigned LineLen = Expansion.size();
    536 
    537     Token PrevPrevTok;
    538     Token PrevTok = Tok;
    539     // Okay, eat this token, getting the next one.
    540     TmpPP.Lex(Tok);
    541 
    542     // Skip all the rest of the tokens that are part of this macro
    543     // instantiation.  It would be really nice to pop up a window with all the
    544     // spelling of the tokens or something.
    545     while (!Tok.is(tok::eof) &&
    546            SM.getExpansionLoc(Tok.getLocation()) == LLoc.first) {
    547       // Insert a newline if the macro expansion is getting large.
    548       if (LineLen > 60) {
    549         Expansion += "<br>";
    550         LineLen = 0;
    551       }
    552 
    553       LineLen -= Expansion.size();
    554 
    555       // If the tokens were already space separated, or if they must be to avoid
    556       // them being implicitly pasted, add a space between them.
    557       if (Tok.hasLeadingSpace() ||
    558           ConcatInfo.AvoidConcat(PrevPrevTok, PrevTok, Tok))
    559         Expansion += ' ';
    560 
    561       // Escape any special characters in the token text.
    562       Expansion += EscapeText(TmpPP.getSpelling(Tok));
    563       LineLen += Expansion.size();
    564 
    565       PrevPrevTok = PrevTok;
    566       PrevTok = Tok;
    567       TmpPP.Lex(Tok);
    568     }
    569 
    570 
    571     // Insert the expansion as the end tag, so that multi-line macros all get
    572     // highlighted.
    573     Expansion = "<span class='expansion'>" + Expansion + "</span></span>";
    574 
    575     HighlightRange(R, LLoc.first, LLoc.second,
    576                    "<span class='macro'>", Expansion.c_str());
    577   }
    578 
    579   // Restore the preprocessor's old state.
    580   TmpPP.setDiagnostics(*OldDiags);
    581   TmpPP.setPragmasEnabled(PragmasPreviouslyEnabled);
    582 }
    583