Home | History | Annotate | Download | only in scripts 
      1 #!/usr/bin/env python
      2 #
      3 # Copyright 2008, Google Inc.
      4 # All rights reserved.
      5 #
      6 # Redistribution and use in source and binary forms, with or without
      7 # modification, are permitted provided that the following conditions are
      8 # met:
      9 #
     10 #     * Redistributions of source code must retain the above copyright
     11 # notice, this list of conditions and the following disclaimer.
     12 #     * Redistributions in binary form must reproduce the above
     13 # copyright notice, this list of conditions and the following disclaimer
     14 # in the documentation and/or other materials provided with the
     15 # distribution.
     16 #     * Neither the name of Google Inc. nor the names of its
     17 # contributors may be used to endorse or promote products derived from
     18 # this software without specific prior written permission.
     19 #
     20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     23 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     24 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     27 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     28 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     29 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     30 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     31 
     32 """pump v0.2.0 - Pretty Useful for Meta Programming.
     33 
     34 A tool for preprocessor meta programming.  Useful for generating
     35 repetitive boilerplate code.  Especially useful for writing C++
     36 classes, functions, macros, and templates that need to work with
     37 various number of arguments.
     38 
     39 USAGE:
     40        pump.py SOURCE_FILE
     41 
     42 EXAMPLES:
     43        pump.py foo.cc.pump
     44          Converts foo.cc.pump to foo.cc.
     45 
     46 GRAMMAR:
     47        CODE ::= ATOMIC_CODE*
     48        ATOMIC_CODE ::= $var ID = EXPRESSION
     49            | $var ID = [[ CODE ]]
     50            | $range ID EXPRESSION..EXPRESSION
     51            | $for ID SEPARATOR [[ CODE ]]
     52            | $($)
     53            | $ID
     54            | $(EXPRESSION)
     55            | $if EXPRESSION [[ CODE ]] ELSE_BRANCH
     56            | [[ CODE ]]
     57            | RAW_CODE
     58        SEPARATOR ::= RAW_CODE | EMPTY
     59        ELSE_BRANCH ::= $else [[ CODE ]]
     60            | $elif EXPRESSION [[ CODE ]] ELSE_BRANCH
     61            | EMPTY
     62        EXPRESSION has Python syntax.
     63 """
     64 
     65 __author__ = 'wan (at] google.com (Zhanyong Wan)'
     66 
     67 import os
     68 import re
     69 import sys
     70 
     71 
     72 TOKEN_TABLE = [
     73     (re.compile(r'\$var\s+'), '$var'),
     74     (re.compile(r'\$elif\s+'), '$elif'),
     75     (re.compile(r'\$else\s+'), '$else'),
     76     (re.compile(r'\$for\s+'), '$for'),
     77     (re.compile(r'\$if\s+'), '$if'),
     78     (re.compile(r'\$range\s+'), '$range'),
     79     (re.compile(r'\$[_A-Za-z]\w*'), '$id'),
     80     (re.compile(r'\$\(\$\)'), '$($)'),
     81     (re.compile(r'\$'), '$'),
     82     (re.compile(r'\[\[\n?'), '[['),
     83     (re.compile(r'\]\]\n?'), ']]'),
     84     ]
     85 
     86 
     87 class Cursor:
     88   """Represents a position (line and column) in a text file."""
     89 
     90   def __init__(self, line=-1, column=-1):
     91     self.line = line
     92     self.column = column
     93 
     94   def __eq__(self, rhs):
     95     return self.line == rhs.line and self.column == rhs.column
     96 
     97   def __ne__(self, rhs):
     98     return not self == rhs
     99 
    100   def __lt__(self, rhs):
    101     return self.line < rhs.line or (
    102         self.line == rhs.line and self.column < rhs.column)
    103 
    104   def __le__(self, rhs):
    105     return self < rhs or self == rhs
    106 
    107   def __gt__(self, rhs):
    108     return rhs < self
    109 
    110   def __ge__(self, rhs):
    111     return rhs <= self
    112 
    113   def __str__(self):
    114     if self == Eof():
    115       return 'EOF'
    116     else:
    117       return '%s(%s)' % (self.line + 1, self.column)
    118 
    119   def __add__(self, offset):
    120     return Cursor(self.line, self.column + offset)
    121 
    122   def __sub__(self, offset):
    123     return Cursor(self.line, self.column - offset)
    124 
    125   def Clone(self):
    126     """Returns a copy of self."""
    127 
    128     return Cursor(self.line, self.column)
    129 
    130 
    131 # Special cursor to indicate the end-of-file.
    132 def Eof():
    133   """Returns the special cursor to denote the end-of-file."""
    134   return Cursor(-1, -1)
    135 
    136 
    137 class Token:
    138   """Represents a token in a Pump source file."""
    139 
    140   def __init__(self, start=None, end=None, value=None, token_type=None):
    141     if start is None:
    142       self.start = Eof()
    143     else:
    144       self.start = start
    145     if end is None:
    146       self.end = Eof()
    147     else:
    148       self.end = end
    149     self.value = value
    150     self.token_type = token_type
    151 
    152   def __str__(self):
    153     return 'Token @%s: \'%s\' type=%s' % (
    154         self.start, self.value, self.token_type)
    155 
    156   def Clone(self):
    157     """Returns a copy of self."""
    158 
    159     return Token(self.start.Clone(), self.end.Clone(), self.value,
    160                  self.token_type)
    161 
    162 
    163 def StartsWith(lines, pos, string):
    164   """Returns True iff the given position in lines starts with 'string'."""
    165 
    166   return lines[pos.line][pos.column:].startswith(string)
    167 
    168 
    169 def FindFirstInLine(line, token_table):
    170   best_match_start = -1
    171   for (regex, token_type) in token_table:
    172     m = regex.search(line)
    173     if m:
    174       # We found regex in lines
    175       if best_match_start < 0 or m.start() < best_match_start:
    176         best_match_start = m.start()
    177         best_match_length = m.end() - m.start()
    178         best_match_token_type = token_type
    179 
    180   if best_match_start < 0:
    181     return None
    182 
    183   return (best_match_start, best_match_length, best_match_token_type)
    184 
    185 
    186 def FindFirst(lines, token_table, cursor):
    187   """Finds the first occurrence of any string in strings in lines."""
    188 
    189   start = cursor.Clone()
    190   cur_line_number = cursor.line
    191   for line in lines[start.line:]:
    192     if cur_line_number == start.line:
    193       line = line[start.column:]
    194     m = FindFirstInLine(line, token_table)
    195     if m:
    196       # We found a regex in line.
    197       (start_column, length, token_type) = m
    198       if cur_line_number == start.line:
    199         start_column += start.column
    200       found_start = Cursor(cur_line_number, start_column)
    201       found_end = found_start + length
    202       return MakeToken(lines, found_start, found_end, token_type)
    203     cur_line_number += 1
    204   # We failed to find str in lines
    205   return None
    206 
    207 
    208 def SubString(lines, start, end):
    209   """Returns a substring in lines."""
    210 
    211   if end == Eof():
    212     end = Cursor(len(lines) - 1, len(lines[-1]))
    213 
    214   if start >= end:
    215     return ''
    216 
    217   if start.line == end.line:
    218     return lines[start.line][start.column:end.column]
    219 
    220   result_lines = ([lines[start.line][start.column:]] +
    221                   lines[start.line + 1:end.line] +
    222                   [lines[end.line][:end.column]])
    223   return ''.join(result_lines)
    224 
    225 
    226 def StripMetaComments(str):
    227   """Strip meta comments from each line in the given string."""
    228 
    229   # First, completely remove lines containing nothing but a meta
    230   # comment, including the trailing \n.
    231   str = re.sub(r'^\s*\$\$.*\n', '', str)
    232 
    233   # Then, remove meta comments from contentful lines.
    234   return re.sub(r'\s*\$\$.*', '', str)
    235 
    236 
    237 def MakeToken(lines, start, end, token_type):
    238   """Creates a new instance of Token."""
    239 
    240   return Token(start, end, SubString(lines, start, end), token_type)
    241 
    242 
    243 def ParseToken(lines, pos, regex, token_type):
    244   line = lines[pos.line][pos.column:]
    245   m = regex.search(line)
    246   if m and not m.start():
    247     return MakeToken(lines, pos, pos + m.end(), token_type)
    248   else:
    249     print 'ERROR: %s expected at %s.' % (token_type, pos)
    250     sys.exit(1)
    251 
    252 
    253 ID_REGEX = re.compile(r'[_A-Za-z]\w*')
    254 EQ_REGEX = re.compile(r'=')
    255 REST_OF_LINE_REGEX = re.compile(r'.*?(?=$|\$\$)')
    256 OPTIONAL_WHITE_SPACES_REGEX = re.compile(r'\s*')
    257 WHITE_SPACE_REGEX = re.compile(r'\s')
    258 DOT_DOT_REGEX = re.compile(r'\.\.')
    259 
    260 
    261 def Skip(lines, pos, regex):
    262   line = lines[pos.line][pos.column:]
    263   m = re.search(regex, line)
    264   if m and not m.start():
    265     return pos + m.end()
    266   else:
    267     return pos
    268 
    269 
    270 def SkipUntil(lines, pos, regex, token_type):
    271   line = lines[pos.line][pos.column:]
    272   m = re.search(regex, line)
    273   if m:
    274     return pos + m.start()
    275   else:
    276     print ('ERROR: %s expected on line %s after column %s.' %
    277            (token_type, pos.line + 1, pos.column))
    278     sys.exit(1)
    279 
    280 
    281 def ParseExpTokenInParens(lines, pos):
    282   def ParseInParens(pos):
    283     pos = Skip(lines, pos, OPTIONAL_WHITE_SPACES_REGEX)
    284     pos = Skip(lines, pos, r'\(')
    285     pos = Parse(pos)
    286     pos = Skip(lines, pos, r'\)')
    287     return pos
    288 
    289   def Parse(pos):
    290     pos = SkipUntil(lines, pos, r'\(|\)', ')')
    291     if SubString(lines, pos, pos + 1) == '(':
    292       pos = Parse(pos + 1)
    293       pos = Skip(lines, pos, r'\)')
    294       return Parse(pos)
    295     else:
    296       return pos
    297 
    298   start = pos.Clone()
    299   pos = ParseInParens(pos)
    300   return MakeToken(lines, start, pos, 'exp')
    301 
    302 
    303 def RStripNewLineFromToken(token):
    304   if token.value.endswith('\n'):
    305     return Token(token.start, token.end, token.value[:-1], token.token_type)
    306   else:
    307     return token
    308 
    309 
    310 def TokenizeLines(lines, pos):
    311   while True:
    312     found = FindFirst(lines, TOKEN_TABLE, pos)
    313     if not found:
    314       yield MakeToken(lines, pos, Eof(), 'code')
    315       return
    316 
    317     if found.start == pos:
    318       prev_token = None
    319       prev_token_rstripped = None
    320     else:
    321       prev_token = MakeToken(lines, pos, found.start, 'code')
    322       prev_token_rstripped = RStripNewLineFromToken(prev_token)
    323 
    324     if found.token_type == '$var':
    325       if prev_token_rstripped:
    326         yield prev_token_rstripped
    327       yield found
    328       id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
    329       yield id_token
    330       pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
    331 
    332       eq_token = ParseToken(lines, pos, EQ_REGEX, '=')
    333       yield eq_token
    334       pos = Skip(lines, eq_token.end, r'\s*')
    335 
    336       if SubString(lines, pos, pos + 2) != '[[':
    337         exp_token = ParseToken(lines, pos, REST_OF_LINE_REGEX, 'exp')
    338         yield exp_token
    339         pos = Cursor(exp_token.end.line + 1, 0)
    340     elif found.token_type == '$for':
    341       if prev_token_rstripped:
    342         yield prev_token_rstripped
    343       yield found
    344       id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
    345       yield id_token
    346       pos = Skip(lines, id_token.end, WHITE_SPACE_REGEX)
    347     elif found.token_type == '$range':
    348       if prev_token_rstripped:
    349         yield prev_token_rstripped
    350       yield found
    351       id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
    352       yield id_token
    353       pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
    354 
    355       dots_pos = SkipUntil(lines, pos, DOT_DOT_REGEX, '..')
    356       yield MakeToken(lines, pos, dots_pos, 'exp')
    357       yield MakeToken(lines, dots_pos, dots_pos + 2, '..')
    358       pos = dots_pos + 2
    359       new_pos = Cursor(pos.line + 1, 0)
    360       yield MakeToken(lines, pos, new_pos, 'exp')
    361       pos = new_pos
    362     elif found.token_type == '$':
    363       if prev_token:
    364         yield prev_token
    365       yield found
    366       exp_token = ParseExpTokenInParens(lines, found.end)
    367       yield exp_token
    368       pos = exp_token.end
    369     elif (found.token_type == ']]' or found.token_type == '$if' or
    370           found.token_type == '$elif' or found.token_type == '$else'):
    371       if prev_token_rstripped:
    372         yield prev_token_rstripped
    373       yield found
    374       pos = found.end
    375     else:
    376       if prev_token:
    377         yield prev_token
    378       yield found
    379       pos = found.end
    380 
    381 
    382 def Tokenize(s):
    383   """A generator that yields the tokens in the given string."""
    384   if s != '':
    385     lines = s.splitlines(True)
    386     for token in TokenizeLines(lines, Cursor(0, 0)):
    387       yield token
    388 
    389 
    390 class CodeNode:
    391   def __init__(self, atomic_code_list=None):
    392     self.atomic_code = atomic_code_list
    393 
    394 
    395 class VarNode:
    396   def __init__(self, identifier=None, atomic_code=None):
    397     self.identifier = identifier
    398     self.atomic_code = atomic_code
    399 
    400 
    401 class RangeNode:
    402   def __init__(self, identifier=None, exp1=None, exp2=None):
    403     self.identifier = identifier
    404     self.exp1 = exp1
    405     self.exp2 = exp2
    406 
    407 
    408 class ForNode:
    409   def __init__(self, identifier=None, sep=None, code=None):
    410     self.identifier = identifier
    411     self.sep = sep
    412     self.code = code
    413 
    414 
    415 class ElseNode:
    416   def __init__(self, else_branch=None):
    417     self.else_branch = else_branch
    418 
    419 
    420 class IfNode:
    421   def __init__(self, exp=None, then_branch=None, else_branch=None):
    422     self.exp = exp
    423     self.then_branch = then_branch
    424     self.else_branch = else_branch
    425 
    426 
    427 class RawCodeNode:
    428   def __init__(self, token=None):
    429     self.raw_code = token
    430 
    431 
    432 class LiteralDollarNode:
    433   def __init__(self, token):
    434     self.token = token
    435 
    436 
    437 class ExpNode:
    438   def __init__(self, token, python_exp):
    439     self.token = token
    440     self.python_exp = python_exp
    441 
    442 
    443 def PopFront(a_list):
    444   head = a_list[0]
    445   a_list[:1] = []
    446   return head
    447 
    448 
    449 def PushFront(a_list, elem):
    450   a_list[:0] = [elem]
    451 
    452 
    453 def PopToken(a_list, token_type=None):
    454   token = PopFront(a_list)
    455   if token_type is not None and token.token_type != token_type:
    456     print 'ERROR: %s expected at %s' % (token_type, token.start)
    457     print 'ERROR: %s found instead' % (token,)
    458     sys.exit(1)
    459 
    460   return token
    461 
    462 
    463 def PeekToken(a_list):
    464   if not a_list:
    465     return None
    466 
    467   return a_list[0]
    468 
    469 
    470 def ParseExpNode(token):
    471   python_exp = re.sub(r'([_A-Za-z]\w*)', r'self.GetValue("\1")', token.value)
    472   return ExpNode(token, python_exp)
    473 
    474 
    475 def ParseElseNode(tokens):
    476   def Pop(token_type=None):
    477     return PopToken(tokens, token_type)
    478 
    479   next = PeekToken(tokens)
    480   if not next:
    481     return None
    482   if next.token_type == '$else':
    483     Pop('$else')
    484     Pop('[[')
    485     code_node = ParseCodeNode(tokens)
    486     Pop(']]')
    487     return code_node
    488   elif next.token_type == '$elif':
    489     Pop('$elif')
    490     exp = Pop('code')
    491     Pop('[[')
    492     code_node = ParseCodeNode(tokens)
    493     Pop(']]')
    494     inner_else_node = ParseElseNode(tokens)
    495     return CodeNode([IfNode(ParseExpNode(exp), code_node, inner_else_node)])
    496   elif not next.value.strip():
    497     Pop('code')
    498     return ParseElseNode(tokens)
    499   else:
    500     return None
    501 
    502 
    503 def ParseAtomicCodeNode(tokens):
    504   def Pop(token_type=None):
    505     return PopToken(tokens, token_type)
    506 
    507   head = PopFront(tokens)
    508   t = head.token_type
    509   if t == 'code':
    510     return RawCodeNode(head)
    511   elif t == '$var':
    512     id_token = Pop('id')
    513     Pop('=')
    514     next = PeekToken(tokens)
    515     if next.token_type == 'exp':
    516       exp_token = Pop()
    517       return VarNode(id_token, ParseExpNode(exp_token))
    518     Pop('[[')
    519     code_node = ParseCodeNode(tokens)
    520     Pop(']]')
    521     return VarNode(id_token, code_node)
    522   elif t == '$for':
    523     id_token = Pop('id')
    524     next_token = PeekToken(tokens)
    525     if next_token.token_type == 'code':
    526       sep_token = next_token
    527       Pop('code')
    528     else:
    529       sep_token = None
    530     Pop('[[')
    531     code_node = ParseCodeNode(tokens)
    532     Pop(']]')
    533     return ForNode(id_token, sep_token, code_node)
    534   elif t == '$if':
    535     exp_token = Pop('code')
    536     Pop('[[')
    537     code_node = ParseCodeNode(tokens)
    538     Pop(']]')
    539     else_node = ParseElseNode(tokens)
    540     return IfNode(ParseExpNode(exp_token), code_node, else_node)
    541   elif t == '$range':
    542     id_token = Pop('id')
    543     exp1_token = Pop('exp')
    544     Pop('..')
    545     exp2_token = Pop('exp')
    546     return RangeNode(id_token, ParseExpNode(exp1_token),
    547                      ParseExpNode(exp2_token))
    548   elif t == '$id':
    549     return ParseExpNode(Token(head.start + 1, head.end, head.value[1:], 'id'))
    550   elif t == '$($)':
    551     return LiteralDollarNode(head)
    552   elif t == '$':
    553     exp_token = Pop('exp')
    554     return ParseExpNode(exp_token)
    555   elif t == '[[':
    556     code_node = ParseCodeNode(tokens)
    557     Pop(']]')
    558     return code_node
    559   else:
    560     PushFront(tokens, head)
    561     return None
    562 
    563 
    564 def ParseCodeNode(tokens):
    565   atomic_code_list = []
    566   while True:
    567     if not tokens:
    568       break
    569     atomic_code_node = ParseAtomicCodeNode(tokens)
    570     if atomic_code_node:
    571       atomic_code_list.append(atomic_code_node)
    572     else:
    573       break
    574   return CodeNode(atomic_code_list)
    575 
    576 
    577 def ParseToAST(pump_src_text):
    578   """Convert the given Pump source text into an AST."""
    579   tokens = list(Tokenize(pump_src_text))
    580   code_node = ParseCodeNode(tokens)
    581   return code_node
    582 
    583 
    584 class Env:
    585   def __init__(self):
    586     self.variables = []
    587     self.ranges = []
    588 
    589   def Clone(self):
    590     clone = Env()
    591     clone.variables = self.variables[:]
    592     clone.ranges = self.ranges[:]
    593     return clone
    594 
    595   def PushVariable(self, var, value):
    596     # If value looks like an int, store it as an int.
    597     try:
    598       int_value = int(value)
    599       if ('%s' % int_value) == value:
    600         value = int_value
    601     except Exception:
    602       pass
    603     self.variables[:0] = [(var, value)]
    604 
    605   def PopVariable(self):
    606     self.variables[:1] = []
    607 
    608   def PushRange(self, var, lower, upper):
    609     self.ranges[:0] = [(var, lower, upper)]
    610 
    611   def PopRange(self):
    612     self.ranges[:1] = []
    613 
    614   def GetValue(self, identifier):
    615     for (var, value) in self.variables:
    616       if identifier == var:
    617         return value
    618 
    619     print 'ERROR: meta variable %s is undefined.' % (identifier,)
    620     sys.exit(1)
    621 
    622   def EvalExp(self, exp):
    623     try:
    624       result = eval(exp.python_exp)
    625     except Exception, e:
    626       print 'ERROR: caught exception %s: %s' % (e.__class__.__name__, e)
    627       print ('ERROR: failed to evaluate meta expression %s at %s' %
    628              (exp.python_exp, exp.token.start))
    629       sys.exit(1)
    630     return result
    631 
    632   def GetRange(self, identifier):
    633     for (var, lower, upper) in self.ranges:
    634       if identifier == var:
    635         return (lower, upper)
    636 
    637     print 'ERROR: range %s is undefined.' % (identifier,)
    638     sys.exit(1)
    639 
    640 
    641 class Output:
    642   def __init__(self):
    643     self.string = ''
    644 
    645   def GetLastLine(self):
    646     index = self.string.rfind('\n')
    647     if index < 0:
    648       return ''
    649 
    650     return self.string[index + 1:]
    651 
    652   def Append(self, s):
    653     self.string += s
    654 
    655 
    656 def RunAtomicCode(env, node, output):
    657   if isinstance(node, VarNode):
    658     identifier = node.identifier.value.strip()
    659     result = Output()
    660     RunAtomicCode(env.Clone(), node.atomic_code, result)
    661     value = result.string
    662     env.PushVariable(identifier, value)
    663   elif isinstance(node, RangeNode):
    664     identifier = node.identifier.value.strip()
    665     lower = int(env.EvalExp(node.exp1))
    666     upper = int(env.EvalExp(node.exp2))
    667     env.PushRange(identifier, lower, upper)
    668   elif isinstance(node, ForNode):
    669     identifier = node.identifier.value.strip()
    670     if node.sep is None:
    671       sep = ''
    672     else:
    673       sep = node.sep.value
    674     (lower, upper) = env.GetRange(identifier)
    675     for i in range(lower, upper + 1):
    676       new_env = env.Clone()
    677       new_env.PushVariable(identifier, i)
    678       RunCode(new_env, node.code, output)
    679       if i != upper:
    680         output.Append(sep)
    681   elif isinstance(node, RawCodeNode):
    682     output.Append(node.raw_code.value)
    683   elif isinstance(node, IfNode):
    684     cond = env.EvalExp(node.exp)
    685     if cond:
    686       RunCode(env.Clone(), node.then_branch, output)
    687     elif node.else_branch is not None:
    688       RunCode(env.Clone(), node.else_branch, output)
    689   elif isinstance(node, ExpNode):
    690     value = env.EvalExp(node)
    691     output.Append('%s' % (value,))
    692   elif isinstance(node, LiteralDollarNode):
    693     output.Append('$')
    694   elif isinstance(node, CodeNode):
    695     RunCode(env.Clone(), node, output)
    696   else:
    697     print 'BAD'
    698     print node
    699     sys.exit(1)
    700 
    701 
    702 def RunCode(env, code_node, output):
    703   for atomic_code in code_node.atomic_code:
    704     RunAtomicCode(env, atomic_code, output)
    705 
    706 
    707 def IsComment(cur_line):
    708   return '//' in cur_line
    709 
    710 
    711 def IsInPreprocessorDirevative(prev_lines, cur_line):
    712   if cur_line.lstrip().startswith('#'):
    713     return True
    714   return prev_lines != [] and prev_lines[-1].endswith('\\')
    715 
    716 
    717 def WrapComment(line, output):
    718   loc = line.find('//')
    719   before_comment = line[:loc].rstrip()
    720   if before_comment == '':
    721     indent = loc
    722   else:
    723     output.append(before_comment)
    724     indent = len(before_comment) - len(before_comment.lstrip())
    725   prefix = indent*' ' + '// '
    726   max_len = 80 - len(prefix)
    727   comment = line[loc + 2:].strip()
    728   segs = [seg for seg in re.split(r'(\w+\W*)', comment) if seg != '']
    729   cur_line = ''
    730   for seg in segs:
    731     if len((cur_line + seg).rstrip()) < max_len:
    732       cur_line += seg
    733     else:
    734       if cur_line.strip() != '':
    735         output.append(prefix + cur_line.rstrip())
    736       cur_line = seg.lstrip()
    737   if cur_line.strip() != '':
    738     output.append(prefix + cur_line.strip())
    739 
    740 
    741 def WrapCode(line, line_concat, output):
    742   indent = len(line) - len(line.lstrip())
    743   prefix = indent*' '  # Prefix of the current line
    744   max_len = 80 - indent - len(line_concat)  # Maximum length of the current line
    745   new_prefix = prefix + 4*' '  # Prefix of a continuation line
    746   new_max_len = max_len - 4  # Maximum length of a continuation line
    747   # Prefers to wrap a line after a ',' or ';'.
    748   segs = [seg for seg in re.split(r'([^,;]+[,;]?)', line.strip()) if seg != '']
    749   cur_line = ''  # The current line without leading spaces.
    750   for seg in segs:
    751     # If the line is still too long, wrap at a space.
    752     while cur_line == '' and len(seg.strip()) > max_len:
    753       seg = seg.lstrip()
    754       split_at = seg.rfind(' ', 0, max_len)
    755       output.append(prefix + seg[:split_at].strip() + line_concat)
    756       seg = seg[split_at + 1:]
    757       prefix = new_prefix
    758       max_len = new_max_len
    759 
    760     if len((cur_line + seg).rstrip()) < max_len:
    761       cur_line = (cur_line + seg).lstrip()
    762     else:
    763       output.append(prefix + cur_line.rstrip() + line_concat)
    764       prefix = new_prefix
    765       max_len = new_max_len
    766       cur_line = seg.lstrip()
    767   if cur_line.strip() != '':
    768     output.append(prefix + cur_line.strip())
    769 
    770 
    771 def WrapPreprocessorDirevative(line, output):
    772   WrapCode(line, ' \\', output)
    773 
    774 
    775 def WrapPlainCode(line, output):
    776   WrapCode(line, '', output)
    777 
    778 
    779 def IsHeaderGuardOrInclude(line):
    780   return (re.match(r'^#(ifndef|define|endif\s*//)\s*[\w_]+\s*$', line) or
    781           re.match(r'^#include\s', line))
    782 
    783 
    784 def WrapLongLine(line, output):
    785   line = line.rstrip()
    786   if len(line) <= 80:
    787     output.append(line)
    788   elif IsComment(line):
    789     if IsHeaderGuardOrInclude(line):
    790       # The style guide made an exception to allow long header guard lines
    791       # and includes.
    792       output.append(line)
    793     else:
    794       WrapComment(line, output)
    795   elif IsInPreprocessorDirevative(output, line):
    796     if IsHeaderGuardOrInclude(line):
    797       # The style guide made an exception to allow long header guard lines
    798       # and includes.
    799       output.append(line)
    800     else:
    801       WrapPreprocessorDirevative(line, output)
    802   else:
    803     WrapPlainCode(line, output)
    804 
    805 
    806 def BeautifyCode(string):
    807   lines = string.splitlines()
    808   output = []
    809   for line in lines:
    810     WrapLongLine(line, output)
    811   output2 = [line.rstrip() for line in output]
    812   return '\n'.join(output2) + '\n'
    813 
    814 
    815 def ConvertFromPumpSource(src_text):
    816   """Return the text generated from the given Pump source text."""
    817   ast = ParseToAST(StripMetaComments(src_text))
    818   output = Output()
    819   RunCode(Env(), ast, output)
    820   return BeautifyCode(output.string)
    821 
    822 
    823 def main(argv):
    824   if len(argv) == 1:
    825     print __doc__
    826     sys.exit(1)
    827 
    828   file_path = argv[-1]
    829   output_str = ConvertFromPumpSource(file(file_path, 'r').read())
    830   if file_path.endswith('.pump'):
    831     output_file_path = file_path[:-5]
    832   else:
    833     output_file_path = '-'
    834   if output_file_path == '-':
    835     print output_str,
    836   else:
    837     output_file = file(output_file_path, 'w')
    838     output_file.write('// This file was GENERATED by command:\n')
    839     output_file.write('//     %s %s\n' %
    840                       (os.path.basename(__file__), os.path.basename(file_path)))
    841     output_file.write('// DO NOT EDIT BY HAND!!!\n\n')
    842     output_file.write(output_str)
    843     output_file.close()
    844 
    845 
    846 if __name__ == '__main__':
    847   main(sys.argv)
    848