Home | History | Annotate | Download | only in tools 
      1 #!/usr/bin/env python
      2 # A tool to parse ASTMatchers.h and update the documentation in
      3 # ../LibASTMatchersReference.html automatically. Run from the
      4 # directory in which this file is located to update the docs.
      5 
      6 import collections
      7 import re
      8 import urllib2
      9 
     10 MATCHERS_FILE = '../../include/clang/ASTMatchers/ASTMatchers.h'
     11 
     12 # Each matcher is documented in one row of the form:
     13 #   result | name | argA
     14 # The subsequent row contains the documentation and is hidden by default,
     15 # becoming visible via javascript when the user clicks the matcher name.
     16 TD_TEMPLATE="""
     17 <tr><td>%(result)s</td><td class="name" onclick="toggle('%(id)s')"><a name="%(id)sAnchor">%(name)s</a></td><td>%(args)s</td></tr>
     18 <tr><td colspan="4" class="doc" id="%(id)s"><pre>%(comment)s</pre></td></tr>
     19 """
     20 
     21 # We categorize the matchers into these three categories in the reference:
     22 node_matchers = {}
     23 narrowing_matchers = {}
     24 traversal_matchers = {}
     25 
     26 # We output multiple rows per matcher if the matcher can be used on multiple
     27 # node types. Thus, we need a new id per row to control the documentation
     28 # pop-up. ids[name] keeps track of those ids.
     29 ids = collections.defaultdict(int)
     30 
     31 # Cache for doxygen urls we have already verified.
     32 doxygen_probes = {}
     33 
     34 def esc(text):
     35   """Escape any html in the given text."""
     36   text = re.sub(r'&', '&amp;', text)
     37   text = re.sub(r'<', '&lt;', text)
     38   text = re.sub(r'>', '&gt;', text)
     39   def link_if_exists(m):
     40     name = m.group(1)
     41     url = 'http://clang.llvm.org/doxygen/classclang_1_1%s.html' % name
     42     if url not in doxygen_probes:
     43       try:
     44         print 'Probing %s...' % url
     45         urllib2.urlopen(url)
     46         doxygen_probes[url] = True
     47       except:
     48         doxygen_probes[url] = False
     49     if doxygen_probes[url]:
     50       return r'Matcher&lt<a href="%s">%s</a>&gt;' % (url, name)
     51     else:
     52       return m.group(0)
     53   text = re.sub(
     54     r'Matcher&lt;([^\*&]+)&gt;', link_if_exists, text)
     55   return text
     56 
     57 def extract_result_types(comment):
     58   """Extracts a list of result types from the given comment.
     59 
     60      We allow annotations in the comment of the matcher to specify what
     61      nodes a matcher can match on. Those comments have the form:
     62        Usable as: Any Matcher | (Matcher<T1>[, Matcher<t2>[, ...]])
     63 
     64      Returns ['*'] in case of 'Any Matcher', or ['T1', 'T2', ...].
     65      Returns the empty list if no 'Usable as' specification could be
     66      parsed.
     67   """
     68   result_types = []
     69   m = re.search(r'Usable as: Any Matcher[\s\n]*$', comment, re.S)
     70   if m:
     71     return ['*']
     72   while True:
     73     m = re.match(r'^(.*)Matcher<([^>]+)>\s*,?[\s\n]*$', comment, re.S)
     74     if not m:
     75       if re.search(r'Usable as:\s*$', comment):
     76         return result_types
     77       else:
     78         return None
     79     result_types += [m.group(2)]
     80     comment = m.group(1)
     81 
     82 def strip_doxygen(comment):
     83   """Returns the given comment without \-escaped words."""
     84   # If there is only a doxygen keyword in the line, delete the whole line.
     85   comment = re.sub(r'^\\[^\s]+\n', r'', comment, flags=re.M)
     86   # Delete the doxygen command and the following whitespace.
     87   comment = re.sub(r'\\[^\s]+\s+', r'', comment)
     88   return comment
     89 
     90 def unify_arguments(args):
     91   """Gets rid of anything the user doesn't care about in the argument list."""
     92   args = re.sub(r'internal::', r'', args)
     93   args = re.sub(r'const\s+', r'', args)
     94   args = re.sub(r'&', r' ', args)
     95   args = re.sub(r'(^|\s)M\d?(\s)', r'\1Matcher<*>\2', args)
     96   return args
     97 
     98 def add_matcher(result_type, name, args, comment, is_dyncast=False):
     99   """Adds a matcher to one of our categories."""
    100   if name == 'id':
    101      # FIXME: Figure out whether we want to support the 'id' matcher.
    102      return
    103   matcher_id = '%s%d' % (name, ids[name])
    104   ids[name] += 1
    105   args = unify_arguments(args)
    106   matcher_html = TD_TEMPLATE % {
    107     'result': esc('Matcher<%s>' % result_type),
    108     'name': name,
    109     'args': esc(args),
    110     'comment': esc(strip_doxygen(comment)),
    111     'id': matcher_id,
    112   }
    113   if is_dyncast:
    114     node_matchers[result_type + name] = matcher_html
    115   # Use a heuristic to figure out whether a matcher is a narrowing or
    116   # traversal matcher. By default, matchers that take other matchers as
    117   # arguments (and are not node matchers) do traversal. We specifically
    118   # exclude known narrowing matchers that also take other matchers as
    119   # arguments.
    120   elif ('Matcher<' not in args or
    121         name in ['allOf', 'anyOf', 'anything', 'unless']):
    122     narrowing_matchers[result_type + name] = matcher_html
    123   else:
    124     traversal_matchers[result_type + name] = matcher_html
    125 
    126 def act_on_decl(declaration, comment, allowed_types):
    127   """Parse the matcher out of the given declaration and comment.
    128 
    129      If 'allowed_types' is set, it contains a list of node types the matcher
    130      can match on, as extracted from the static type asserts in the matcher
    131      definition.
    132   """
    133   if declaration.strip():
    134     # Node matchers are defined by writing:
    135     #   VariadicDynCastAllOfMatcher<ResultType, ArgumentType> name;
    136     m = re.match(r""".*VariadicDynCastAllOfMatcher\s*<
    137                        \s*([^\s,]+)\s*,
    138                        \s*([^\s>]+)\s*>
    139                        \s*([^\s;]+)\s*;\s*$""", declaration, flags=re.X)
    140     if m:
    141       result, inner, name = m.groups()
    142       add_matcher(result, name, 'Matcher<%s>...' % inner,
    143                   comment, is_dyncast=True)
    144       return
    145 
    146     # Parse the various matcher definition macros.
    147     m = re.match(r"""^\s*AST_(POLYMORPHIC_)?MATCHER(_P)?(.?)\(
    148                        (?:\s*([^\s,]+)\s*,)?
    149                           \s*([^\s,]+)\s*
    150                        (?:,\s*([^\s,]+)\s*
    151                           ,\s*([^\s,]+)\s*)?
    152                        (?:,\s*([^\s,]+)\s*
    153                           ,\s*([^\s,]+)\s*)?
    154                       \)\s*{\s*$""", declaration, flags=re.X)
    155     if m:
    156       p, n, result, name = m.groups()[1:5]
    157       args = m.groups()[5:]
    158       if not result:
    159         if not allowed_types:
    160           raise Exception('Did not find allowed result types for: %s' % name)
    161         result_types = allowed_types
    162       else:
    163         result_types = [result]
    164       if n not in ['', '2']:
    165         raise Exception('Cannot parse "%s"' % declaration)
    166       args = ', '.join('%s %s' % (args[i], args[i+1])
    167                        for i in range(0, len(args), 2) if args[i])
    168       for result_type in result_types:
    169         add_matcher(result_type, name, args, comment)
    170       return
    171 
    172     # Parse free standing matcher functions, like:
    173     #   Matcher<ResultType> Name(Matcher<ArgumentType> InnerMatcher) {
    174     m = re.match(r"""^\s*(.*)\s+
    175                      ([^\s\(]+)\s*\(
    176                      (.*)
    177                      \)\s*{""", declaration, re.X)
    178     if m:
    179       result, name, args = m.groups()
    180       args = ', '.join(p.strip() for p in args.split(','))
    181       m = re.match(r'.*\s+internal::Matcher<([^>]+)>$', result)
    182       if m:
    183         result_types = [m.group(1)]
    184       else:
    185         result_types = extract_result_types(comment)
    186       if not result_types:
    187         if not comment:
    188           # Only overloads don't have their own doxygen comments; ignore those.
    189           print 'Ignoring "%s"' % name
    190         else:
    191           print 'Cannot determine result type for "%s"' % name
    192       else:
    193         for result_type in result_types:
    194           add_matcher(result_type, name, args, comment)
    195     else:
    196       print '*** Unparsable: "' + declaration + '" ***'
    197 
    198 def sort_table(matcher_type, matcher_map):
    199   """Returns the sorted html table for the given row map."""
    200   table = ''
    201   for key in sorted(matcher_map.keys()):
    202     table += matcher_map[key] + '\n'
    203   return ('<!-- START_%(type)s_MATCHERS -->\n' +
    204           '%(table)s' + 
    205           '<!--END_%(type)s_MATCHERS -->') % {
    206     'type': matcher_type,
    207     'table': table,
    208   }
    209 
    210 # Parse the ast matchers.
    211 # We alternate between two modes:
    212 # body = True: We parse the definition of a matcher. We need
    213 #   to parse the full definition before adding a matcher, as the
    214 #   definition might contain static asserts that specify the result
    215 #   type.
    216 # body = False: We parse the comments and declaration of the matcher.
    217 comment = ''
    218 declaration = ''
    219 allowed_types = []
    220 body = False
    221 for line in open(MATCHERS_FILE).read().splitlines():
    222   if body:
    223     if line.strip() and line[0] == '}':
    224       if declaration:
    225         act_on_decl(declaration, comment, allowed_types)
    226         comment = ''
    227         declaration = ''
    228         allowed_types = []
    229       body = False
    230     else:
    231       m = re.search(r'is_base_of<([^,]+), NodeType>', line)
    232       if m and m.group(1):
    233         allowed_types += [m.group(1)]
    234     continue
    235   if line.strip() and line.lstrip()[0] == '/':
    236     comment += re.sub(r'/+\s?', '', line) + '\n'
    237   else:
    238     declaration += ' ' + line
    239     if ((not line.strip()) or 
    240         line.rstrip()[-1] == ';' or
    241         line.rstrip()[-1] == '{'):
    242       if line.strip() and line.rstrip()[-1] == '{':
    243         body = True
    244       else:
    245         act_on_decl(declaration, comment, allowed_types)
    246         comment = ''
    247         declaration = ''
    248         allowed_types = []
    249 
    250 node_matcher_table = sort_table('DECL', node_matchers)
    251 narrowing_matcher_table = sort_table('NARROWING', narrowing_matchers)
    252 traversal_matcher_table = sort_table('TRAVERSAL', traversal_matchers)
    253 
    254 reference = open('../LibASTMatchersReference.html').read()
    255 reference = re.sub(r'<!-- START_DECL_MATCHERS.*END_DECL_MATCHERS -->',
    256                    '%s', reference, flags=re.S) % node_matcher_table
    257 reference = re.sub(r'<!-- START_NARROWING_MATCHERS.*END_NARROWING_MATCHERS -->',
    258                    '%s', reference, flags=re.S) % narrowing_matcher_table
    259 reference = re.sub(r'<!-- START_TRAVERSAL_MATCHERS.*END_TRAVERSAL_MATCHERS -->',
    260                    '%s', reference, flags=re.S) % traversal_matcher_table
    261 
    262 with open('../LibASTMatchersReference.html', 'w') as output:
    263   output.write(reference)
    264 
    265