Home | History | Annotate | Download | only in Lex 
      1 //===--- TokenLexer.h - Lex from a token buffer -----------------*- C++ -*-===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 // This file defines the TokenLexer interface.
     11 //
     12 //===----------------------------------------------------------------------===//
     13 
     14 #ifndef LLVM_CLANG_LEX_TOKENLEXER_H
     15 #define LLVM_CLANG_LEX_TOKENLEXER_H
     16 
     17 #include "clang/Basic/SourceLocation.h"
     18 
     19 namespace clang {
     20   class MacroInfo;
     21   class Preprocessor;
     22   class Token;
     23   class MacroArgs;
     24 
     25 /// TokenLexer - This implements a lexer that returns tokens from a macro body
     26 /// or token stream instead of lexing from a character buffer.  This is used for
     27 /// macro expansion and _Pragma handling, for example.
     28 ///
     29 class TokenLexer {
     30   /// Macro - The macro we are expanding from.  This is null if expanding a
     31   /// token stream.
     32   ///
     33   MacroInfo *Macro;
     34 
     35   /// ActualArgs - The actual arguments specified for a function-like macro, or
     36   /// null.  The TokenLexer owns the pointed-to object.
     37   MacroArgs *ActualArgs;
     38 
     39   /// PP - The current preprocessor object we are expanding for.
     40   ///
     41   Preprocessor &PP;
     42 
     43   /// Tokens - This is the pointer to an array of tokens that the macro is
     44   /// defined to, with arguments expanded for function-like macros.  If this is
     45   /// a token stream, these are the tokens we are returning.  This points into
     46   /// the macro definition we are lexing from, a cache buffer that is owned by
     47   /// the preprocessor, or some other buffer that we may or may not own
     48   /// (depending on OwnsTokens).
     49   /// Note that if it points into Preprocessor's cache buffer, the Preprocessor
     50   /// may update the pointer as needed.
     51   const Token *Tokens;
     52   friend class Preprocessor;
     53 
     54   /// NumTokens - This is the length of the Tokens array.
     55   ///
     56   unsigned NumTokens;
     57 
     58   /// CurToken - This is the next token that Lex will return.
     59   ///
     60   unsigned CurToken;
     61 
     62   /// ExpandLocStart/End - The source location range where this macro was
     63   /// expanded.
     64   SourceLocation ExpandLocStart, ExpandLocEnd;
     65 
     66   /// \brief Source location pointing at the source location entry chunk that
     67   /// was reserved for the current macro expansion.
     68   SourceLocation MacroExpansionStart;
     69 
     70   /// \brief The offset of the macro expansion in the
     71   /// "source location address space".
     72   unsigned MacroStartSLocOffset;
     73 
     74   /// \brief Location of the macro definition.
     75   SourceLocation MacroDefStart;
     76   /// \brief Length of the macro definition.
     77   unsigned MacroDefLength;
     78 
     79   /// Lexical information about the expansion point of the macro: the identifier
     80   /// that the macro expanded from had these properties.
     81   bool AtStartOfLine : 1;
     82   bool HasLeadingSpace : 1;
     83 
     84   // NextTokGetsSpace - When this is true, the next token appended to the
     85   // output list during function argument expansion will get a leading space,
     86   // regardless of whether it had one to begin with or not. This is used for
     87   // placemarker support. If still true after function argument expansion, the
     88   // leading space will be applied to the first token following the macro
     89   // expansion.
     90   bool NextTokGetsSpace : 1;
     91 
     92   /// OwnsTokens - This is true if this TokenLexer allocated the Tokens
     93   /// array, and thus needs to free it when destroyed.  For simple object-like
     94   /// macros (for example) we just point into the token buffer of the macro
     95   /// definition, we don't make a copy of it.
     96   bool OwnsTokens : 1;
     97 
     98   /// DisableMacroExpansion - This is true when tokens lexed from the TokenLexer
     99   /// should not be subject to further macro expansion.
    100   bool DisableMacroExpansion : 1;
    101 
    102   TokenLexer(const TokenLexer &) = delete;
    103   void operator=(const TokenLexer &) = delete;
    104 public:
    105   /// Create a TokenLexer for the specified macro with the specified actual
    106   /// arguments.  Note that this ctor takes ownership of the ActualArgs pointer.
    107   /// ILEnd specifies the location of the ')' for a function-like macro or the
    108   /// identifier for an object-like macro.
    109   TokenLexer(Token &Tok, SourceLocation ILEnd, MacroInfo *MI,
    110              MacroArgs *ActualArgs, Preprocessor &pp)
    111     : Macro(nullptr), ActualArgs(nullptr), PP(pp), OwnsTokens(false) {
    112     Init(Tok, ILEnd, MI, ActualArgs);
    113   }
    114 
    115   /// Init - Initialize this TokenLexer to expand from the specified macro
    116   /// with the specified argument information.  Note that this ctor takes
    117   /// ownership of the ActualArgs pointer.  ILEnd specifies the location of the
    118   /// ')' for a function-like macro or the identifier for an object-like macro.
    119   void Init(Token &Tok, SourceLocation ILEnd, MacroInfo *MI,
    120             MacroArgs *ActualArgs);
    121 
    122   /// Create a TokenLexer for the specified token stream.  If 'OwnsTokens' is
    123   /// specified, this takes ownership of the tokens and delete[]'s them when
    124   /// the token lexer is empty.
    125   TokenLexer(const Token *TokArray, unsigned NumToks, bool DisableExpansion,
    126              bool ownsTokens, Preprocessor &pp)
    127     : Macro(nullptr), ActualArgs(nullptr), PP(pp), OwnsTokens(false) {
    128     Init(TokArray, NumToks, DisableExpansion, ownsTokens);
    129   }
    130 
    131   /// Init - Initialize this TokenLexer with the specified token stream.
    132   /// This does not take ownership of the specified token vector.
    133   ///
    134   /// DisableExpansion is true when macro expansion of tokens lexed from this
    135   /// stream should be disabled.
    136   void Init(const Token *TokArray, unsigned NumToks,
    137             bool DisableMacroExpansion, bool OwnsTokens);
    138 
    139   ~TokenLexer() { destroy(); }
    140 
    141   /// isNextTokenLParen - If the next token lexed will pop this macro off the
    142   /// expansion stack, return 2.  If the next unexpanded token is a '(', return
    143   /// 1, otherwise return 0.
    144   unsigned isNextTokenLParen() const;
    145 
    146   /// Lex - Lex and return a token from this macro stream.
    147   bool Lex(Token &Tok);
    148 
    149   /// isParsingPreprocessorDirective - Return true if we are in the middle of a
    150   /// preprocessor directive.
    151   bool isParsingPreprocessorDirective() const;
    152 
    153 private:
    154   void destroy();
    155 
    156   /// isAtEnd - Return true if the next lex call will pop this macro off the
    157   /// include stack.
    158   bool isAtEnd() const {
    159     return CurToken == NumTokens;
    160   }
    161 
    162   /// PasteTokens - Tok is the LHS of a ## operator, and CurToken is the ##
    163   /// operator.  Read the ## and RHS, and paste the LHS/RHS together.  If there
    164   /// are is another ## after it, chomp it iteratively.  Return the result as
    165   /// Tok.  If this returns true, the caller should immediately return the
    166   /// token.
    167   bool PasteTokens(Token &Tok);
    168 
    169   /// Expand the arguments of a function-like macro so that we can quickly
    170   /// return preexpanded tokens from Tokens.
    171   void ExpandFunctionArguments();
    172 
    173   /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
    174   /// together to form a comment that comments out everything in the current
    175   /// macro, other active macros, and anything left on the current physical
    176   /// source line of the expanded buffer.  Handle this by returning the
    177   /// first token on the next line.
    178   void HandleMicrosoftCommentPaste(Token &Tok);
    179 
    180   /// \brief If \p loc is a FileID and points inside the current macro
    181   /// definition, returns the appropriate source location pointing at the
    182   /// macro expansion source location entry.
    183   SourceLocation getExpansionLocForMacroDefLoc(SourceLocation loc) const;
    184 
    185   /// \brief Creates SLocEntries and updates the locations of macro argument
    186   /// tokens to their new expanded locations.
    187   ///
    188   /// \param ArgIdSpellLoc the location of the macro argument id inside the
    189   /// macro definition.
    190   void updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
    191                                   Token *begin_tokens, Token *end_tokens);
    192 
    193   /// Remove comma ahead of __VA_ARGS__, if present, according to compiler
    194   /// dialect settings.  Returns true if the comma is removed.
    195   bool MaybeRemoveCommaBeforeVaArgs(SmallVectorImpl<Token> &ResultToks,
    196                                     bool HasPasteOperator,
    197                                     MacroInfo *Macro, unsigned MacroArgNo,
    198                                     Preprocessor &PP);
    199 
    200   void PropagateLineStartLeadingSpaceInfo(Token &Result);
    201 };
    202 
    203 }  // end namespace clang
    204 
    205 #endif
    206