Home | History | Annotate | Download | only in syntax
      1 /*
      2  * Copyright (C) 2010 Google Inc.
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  * http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17  /* This is the JSilver grammar fed into SableCC to generate the parser. */
     18 
     19 // Java package of generated code.
     20 Package
     21   com.google.clearsilver.jsilver.syntax;
     22 
     23 /***** Stage 1: The Lexer
     24  *
     25  * The lexer breaks the inbound text stream in to a sequence of tokens.
     26  *
     27  * SableCC will generate a wrapper class for each type of token.
     28  *
     29  * Anything outside of <?cs and ?> will be returned as a TData token.
     30  * Anything in between (including) <?cs and ?> will be broken down
     31  * into a finer grained set of tokens.
     32  *
     33  * For example: "Hello <?cs var:person.name ?>!"
     34  * Results in:
     35  *  TData        "Hello "
     36  *  TCsOpen      "<?cs"
     37  *  TWhiteSpace  " "
     38  *  TVar         "var"
     39  *  TColon       ":"
     40  *  TWord        "person.name"
     41  *  TWhitspace   " "
     42  *  TCsClose     "?>"
     43  *  TData        "!"
     44  */
     45 
     46 /* Constants to be used elsewhere. */
     47 Helpers
     48 
     49   // These aren't actually tokens, but can be used by tokens. Avoids the
     50   // need for magic numbers in the token definitions.
     51   alphanumeric = (['a' .. 'z'] | ['A' .. 'Z'] | ['0' .. '9']);
     52   alpha        = (['a' .. 'z'] | ['A' .. 'Z']);
     53   all = [0 .. 0xFFFF];
     54   tab = 9;
     55   cr = 13;
     56   lf = 10;
     57   whitespace = (tab | cr | lf | ' ');
     58 
     59 /* States of the lexer. */
     60 States
     61 
     62   content,     // Anything outside of <?cs and ?>.
     63   command,     // ClearSilver command: "<?cs var:".
     64   args,        // Args to command: "some.variable=3 ?>"
     65   comment;     // Inside a <?cs # comment ?>.
     66 
     67 /* Tokens and state transitions. */
     68 Tokens
     69 
     70   // In the 'content' state, treat everything as a chunk of data,
     71   // up to the sequence '<?cs '.
     72   // Note, because the lexer has to read 5 characters '<?cs ' before
     73   // knowing the definite state, it needs to be supplied a
     74   // PushbackReader() with a buffer of at least 5.
     75   {content} data = (        [all - '<']
     76                    | '<'    [all - '?']
     77                    | '<?'   [all - 'c']
     78                    | '<?c'  [all - 's']
     79                    | '<?cs' [[[[all - ' '] - cr] - lf] - tab])+;
     80 
     81   {comment} comment = (        [all - '?']
     82                       | '?'    [all - '>'] )+; // All up to ?>
     83 
     84   // In the 'clearsilver' state, break each keyword, operator
     85   // and symbol into it's own token.
     86   {command} var           = 'var';
     87   {command} lvar          = 'lvar';
     88   {command} evar          = 'evar';
     89   {command} uvar          = 'uvar';
     90   {command} set           = 'set';
     91   {command} if            = 'if';
     92   {command} else_if       = ('elif' | 'elseif');
     93   {command} else          = 'else';
     94   {command} with          = 'with';
     95   {command} escape        = 'escape';
     96   {command} autoescape    = 'autoescape';
     97   {command} loop          = 'loop';
     98   {command} each          = 'each';
     99   {command} alt           = 'alt';
    100   {command} name          = 'name';
    101   {command} def           = 'def';
    102   {command} call          = 'call';
    103   {command} include       = 'include';
    104   {command} linclude      = 'linclude';
    105   {command} content_type  = 'content-type';
    106   {command} inline        = 'inline';
    107 
    108   {args} comma         = ',';
    109   {args} bang          = '!';
    110   {args} assignment    = '=';
    111   {args} eq            = '==';
    112   {args} ne            = '!=';
    113   {args} lt            = '<';
    114   {args} gt            = '>';
    115   {args} lte           = '<=';
    116   {args} gte           = '>=';
    117   {args} and           = '&&';
    118   {args} or            = '||';
    119   {args} string        = ('"' [all - '"']* '"' | ''' [all - ''']* ''');
    120   {args} hash          = '#';
    121   {args} plus          = '+';
    122   {args} minus         = '-';
    123   {args} star          = '*';
    124   {args} percent       = '%';
    125   {args} bracket_open  = '[';
    126   {args} bracket_close = ']';
    127   {args} paren_open    = '(';
    128   {args} paren_close   = ')';
    129   {args} dot           = '.';
    130   {args} dollar        = '$';
    131   {args} question      = '?';
    132   {args} dec_number    = (['0' .. '9'])+;
    133   {args} hex_number    = ('0x' | '0X') (['0' .. '9'] | ['a' .. 'f'] | ['A' .. 'F'])+;
    134   {args} word          = (alphanumeric | '_')+;
    135   {args} arg_whitespace = whitespace+;
    136 
    137   // Tokens that are valid in multiple states
    138   {command, args} slash         = '/'; // means divide or end command.
    139 
    140   // State transitions.
    141   {content->command}                                  cs_open        = '<?cs' whitespace+;
    142   {command->comment}                                  comment_start  = '#';
    143   {command->args}                                     command_delimiter = ':' | whitespace;
    144   {command->args}                                     hard_delimiter = '!' | whitespace;
    145   {command->content, args->content, comment->content} cs_close       = whitespace* '?>';
    146 
    147 /***** Stage 2: The Parser
    148  *
    149  * Below is a BNF-like grammar of how the tokens are assembled.
    150  * The generate parser will read the token stream and build a
    151  * tree of nodes representing the structure.
    152  *
    153  * This is the Concrete Syntax Tree (CST).
    154  *
    155  * Though this provides access to the underlying syntax tree, the
    156  * resulting tree would be quite tricky to work with from code as
    157  * it's heavily loaded with syntax specifics and parser tricks...
    158  *
    159  * So, the CST also contains transformation rules ({->x}) to
    160  * convert it into a much simpler Abstract Syntax Tree (AST),
    161  * which is defined in stage 3.
    162  */
    163 
    164 /* Tokens from the lexer that the parser doesn't care about. */
    165 Ignored Tokens
    166 
    167   arg_whitespace;
    168 
    169 /* Concrete syntax tree. */
    170 Productions
    171 
    172   // Overall template structure...
    173 
    174   grammar {->command}
    175     = commands
    176               {->commands.command}
    177     ;
    178 
    179   commands {->command}
    180     = {none}
    181               {->New command.noop()}
    182     | {one}   command
    183               {->command.command}
    184     | {many}  command [more]:command+
    185               {->New command.multiple([command.command, more.command])}
    186     ;
    187 
    188   command {->command}
    189 
    190     = {data}  data
    191               // Anything outside of <?cs ?> tag
    192               {->New command.data(data)}
    193 
    194     | {comment} cs_open comment_start comment? cs_close
    195               // <?cs # comment ?>
    196               {->New command.comment(New position.cs_open(cs_open),comment)}
    197 
    198     | {var}   cs_open var command_delimiter expression_list cs_close
    199               // <?cs var:x ?>
    200               {->New command.var(
    201                    New position.cs_open(cs_open),
    202                    New expression.sequence([expression_list.expression]))}
    203 
    204     | {lvar}  cs_open lvar command_delimiter expression_list cs_close
    205               // <?cs lvar:x ?>
    206               {->New command.lvar(
    207                    New position.cs_open(cs_open),
    208                    New expression.sequence([expression_list.expression]))}
    209 
    210     | {evar}  cs_open evar command_delimiter expression_list cs_close
    211               // <?cs evar:x ?>
    212               {->New command.evar(
    213                    New position.cs_open(cs_open),
    214                    New expression.sequence([expression_list.expression]))}
    215 
    216     | {uvar}  cs_open uvar command_delimiter expression_list cs_close
    217               // <?cs uvar:x ?>
    218               {->New command.uvar(
    219                    New position.cs_open(cs_open),
    220                    New expression.sequence([expression_list.expression]))}
    221 
    222     | {set}   cs_open set command_delimiter variable assignment expression cs_close
    223               // <?cs set:x = y ?>
    224               {->New command.set(
    225                    New position.cs_open(cs_open),
    226                    variable.variable,
    227                    expression.expression)}
    228 
    229     | {name}  cs_open name command_delimiter variable cs_close
    230               // <?cs name:x ?>
    231               {->New command.name(
    232                    New position.cs_open(cs_open),
    233                    variable.variable)}
    234 
    235     | {escape} cs_open escape command_delimiter expression cs_close
    236               commands
    237               [i1]:cs_open slash [i3]:escape [i2]:cs_close
    238               // <?cs escape:"html" ?>...<?cs /escape?>
    239               {->New command.escape(
    240                    New position.cs_open(cs_open),
    241                    expression.expression,
    242                    commands.command)}
    243 
    244     | {autoescape} cs_open autoescape command_delimiter expression cs_close
    245               commands
    246               [i1]:cs_open slash [i3]:autoescape [i2]:cs_close
    247               // <?cs autoescape:"html" ?>...<?cs /autoescape?>
    248               {->New command.autoescape(
    249                    New position.cs_open(cs_open),
    250                    expression.expression,
    251                    commands.command)}
    252 
    253     | {with}  cs_open with command_delimiter variable assignment expression cs_close
    254               commands
    255               [i1]:cs_open slash [i3]:with [i2]:cs_close
    256               // <?cs with:x=y ?>...<?cs /with?>
    257               {->New command.with(
    258                   New position.cs_open(cs_open),
    259                   variable.variable,
    260                   expression.expression,
    261                   commands.command)}
    262 
    263     | {loop_to}  cs_open loop command_delimiter variable assignment expression cs_close
    264               commands
    265               [i1]:cs_open slash [i3]:loop [i2]:cs_close
    266               // <?cs loop:x=20 ?>...<?cs /loop ?>
    267               {->New command.loop_to(
    268                   New position.cs_open(cs_open),
    269                   variable.variable,
    270                   expression.expression,
    271                   commands.command)}
    272 
    273     | {loop}  cs_open loop command_delimiter variable assignment
    274               [start]:expression comma [end]:expression cs_close
    275               commands
    276               [i1]:cs_open slash [i3]:loop [i2]:cs_close
    277               // <?cs loop:x=1,20 ?>...<?cs /loop ?>
    278               {->New command.loop(
    279                   New position.cs_open(cs_open),
    280                   variable.variable,
    281                   start.expression,
    282                   end.expression,
    283                   commands.command)}
    284 
    285     | {loop_inc} cs_open loop command_delimiter variable assignment
    286               [start]:expression comma
    287               [end]:expression [i3]:comma [increment]:expression cs_close
    288               commands [i1]:cs_open slash [i4]:loop [i2]:cs_close
    289               // <?cs loop:x=1,20,5 ?>...<?cs /loop ?>
    290               {->New command.loop_inc(
    291                   New position.cs_open(cs_open),
    292                   variable.variable,
    293                   start.expression,
    294                   end.expression,
    295                   increment.expression,
    296                   commands.command)}
    297 
    298     | {each}  cs_open each command_delimiter variable assignment expression cs_close
    299               commands
    300               [i1]:cs_open slash [i3]:each [i2]:cs_close
    301               // <?cs each:x=some.thing ?>...<?cs /each ?>
    302               {->New command.each(
    303                   New position.cs_open(cs_open),
    304                   variable.variable,
    305                   expression.expression,
    306                   commands.command)}
    307 
    308     | {alt}   cs_open alt command_delimiter expression cs_close
    309               commands
    310               [i1]:cs_open slash [i3]:alt [i2]:cs_close
    311               // <?cs alt:some.thing ?>...<?cs /alt ?>
    312               {->New command.alt(
    313                   New position.cs_open(cs_open),
    314                   expression.expression,
    315                   commands.command)}
    316 
    317     | {def}   cs_open def command_delimiter multipart_word paren_open variable_list?
    318               paren_close cs_close commands
    319               [i1]:cs_open slash [i3]:def [i2]:cs_close
    320               // <?cs def:some.macro(arg,arg) ?>...<?cs /def ?>
    321               {->New command.def(
    322                   New position.cs_open(cs_open),
    323                   [multipart_word.word],
    324                   [variable_list.variable],
    325                   commands.command)}
    326 
    327     | {call}  cs_open call command_delimiter multipart_word paren_open expression_list?
    328               paren_close cs_close
    329               // <?cs call:some.macro(arg,arg) ?>
    330               {->New command.call(
    331                   New position.cs_open(cs_open),
    332                   [multipart_word.word],
    333                   [expression_list.expression])}
    334 
    335     | {if}    if_block
    336               {->if_block.command}
    337 
    338     | {include} cs_open include command_delimiter expression cs_close
    339               // <?cs include:x ?>
    340               {->New command.include(
    341                   New position.cs_open(cs_open),
    342                   expression.expression)}
    343 
    344     | {hard_include} cs_open include hard_delimiter expression cs_close
    345               // <?cs include!x ?>
    346               {->New command.hard_include(
    347                   New position.cs_open(cs_open),
    348                   expression.expression)}
    349 
    350     | {linclude} cs_open linclude command_delimiter expression cs_close
    351               // <?cs linclude:x ?>
    352               {->New command.linclude(
    353                   New position.cs_open(cs_open),
    354                   expression.expression)}
    355 
    356     | {hard_linclude} cs_open linclude hard_delimiter expression cs_close
    357               // <?cs linclude!x ?>
    358               {->New command.hard_linclude(
    359                   New position.cs_open(cs_open),
    360                   expression.expression)}
    361 
    362     | {content_type} cs_open content_type command_delimiter string cs_close
    363               // <?cs content-type:"html" ?>
    364               {->New command.content_type(
    365                   New position.cs_open(cs_open),
    366                   string)}
    367 
    368     | {inline} cs_open inline cs_close
    369               commands
    370               [i1]:cs_open slash [i3]:inline [i2]:cs_close
    371               // <?cs inline ?>...<?cs /inline?>
    372               {->New command.inline(
    373                    New position.cs_open(cs_open),
    374                    commands.command)}
    375 
    376     ;
    377 
    378   multipart_word {->word*}
    379     = {bit} word
    380               {->[word]}
    381     | {m} multipart_word dot word
    382               {->[multipart_word.word, word]}
    383     ;
    384 
    385   variable_list {->variable*}
    386     = {single} variable
    387               {->[variable.variable]}
    388     | {multiple} variable_list comma variable
    389               {->[variable_list.variable, variable.variable]}
    390     ;
    391 
    392   expression_list {->expression*}
    393     = {single} expression
    394               {->[expression.expression]}
    395     | {multiple} expression_list comma expression
    396               {->[expression_list.expression, expression.expression]}
    397     ;
    398 
    399   // If/ElseIf/Else block...
    400 
    401   if_block {->command}
    402     =           cs_open if command_delimiter expression cs_close
    403                 commands
    404                 else_if_block
    405                 // <?cs if:x.y ?> (commands, then optional else_if_block)
    406                 {->New command.if(
    407                     New position.cs_open(cs_open),
    408                     expression.expression,
    409                     commands.command,
    410                     else_if_block.command)}
    411     ;
    412 
    413   // ElseIf statements get transformed into nested if/else blocks to simplify
    414   // final AST.
    415   else_if_block {->command}
    416     = {present} cs_open else_if command_delimiter expression cs_close
    417                 commands
    418                 else_if_block
    419                 // <?cs elif:x.y ?> (recurses)
    420                 {->New command.if(
    421                     New position.cs_open(cs_open),
    422                     expression.expression,
    423                     commands.command,
    424                     else_if_block.command)}
    425     | {missing} else_block
    426                 {->else_block.command}
    427     ;
    428 
    429   else_block {->command}
    430     = {present} cs_open else cs_close
    431                 commands
    432                 end_if_block
    433                 // <?cs else ?> (followed by end_if_block)
    434                 {->commands.command}
    435     | {skip}    end_if_block
    436                 {->New command.noop()}
    437     ;
    438 
    439   end_if_block
    440     =           cs_open slash if cs_close
    441                 // <?cs /if ?>
    442     ;
    443 
    444   // Expression language...
    445 
    446   // The multiple levels allow the parser to build a tree based on operator
    447   // precedence. The higher the level in the tree, the lower the precedence.
    448 
    449   expression {->expression}
    450     = {or} [left]:expression or [right]:and_expression    // x.y || a.b
    451               {->New expression.or(left.expression, right.expression)}
    452     | {and_expression} [value]:and_expression             // x.y
    453               {->value.expression}
    454     ;
    455 
    456   and_expression {->expression}
    457     = {and}  [left]:and_expression and [right]:equality   // x.y && a.b
    458               {->New expression.and(left.expression, right.expression)}
    459     | {equality} [value]:equality                         // x.y
    460               {->value.expression}
    461     ;
    462 
    463   equality {->expression}
    464     = {eq}  [left]:equality eq [right]:comparison         // x.y == a.b
    465               {->New expression.eq(left.expression, right.expression)}
    466     | {ne}  [left]:equality ne [right]:comparison         // x.y != a.b
    467               {->New expression.ne(left.expression, right.expression)}
    468     | {comparison} [value]:comparison                  // x.y
    469               {->value.expression}
    470     ;
    471 
    472   comparison {->expression}
    473     = {lt}  [left]:comparison lt  [right]:add_subtract         // x.y < a.b
    474               {->New expression.lt(left.expression, right.expression)}
    475     | {gt}  [left]:comparison gt  [right]:add_subtract         // x.y > a.b
    476               {->New expression.gt(left.expression, right.expression)}
    477     | {lte} [left]:comparison lte [right]:add_subtract         // x.y <= a.b
    478               {->New expression.lte(left.expression, right.expression)}
    479     | {gte} [left]:comparison gte [right]:add_subtract         // x.y >= a.b
    480               {->New expression.gte(left.expression, right.expression)}
    481     | {add_subtract} [value]:add_subtract                      // x.y
    482               {->value.expression}
    483     ;
    484 
    485   add_subtract {->expression}
    486     = {add} [left]:add_subtract plus [right]:factor        // x.y + a.b
    487               {->New expression.add(left.expression, right.expression)}
    488     | {subtract} [left]:add_subtract minus [right]:factor  // x.y - a.b
    489               {->New expression.subtract(left.expression, right.expression)}
    490     | {factor} [value]:factor                              // x.y
    491               {->value.expression}
    492     ;
    493 
    494   factor {->expression}
    495     = {multiply} [left]:factor star [right]:value   // x.y * a.b
    496               {->New expression.multiply(left.expression, right.expression)}
    497     | {divide} [left]:factor slash [right]:value    // x.y / a.b
    498               {->New expression.divide(left.expression, right.expression)}
    499     | {modulo} [left]:factor percent [right]:value  // x.y % a.b
    500               {->New expression.modulo(left.expression, right.expression)}
    501     | {value} value                                 // x.y
    502               {->value.expression}
    503     ;
    504 
    505   value {->expression}
    506     = {variable} variable          // x.y
    507               {->New expression.variable(variable.variable)}
    508     | {string} string              // "hello"
    509               {->New expression.string(string)}
    510     | {number} number              // 123
    511               {->number.expression}
    512     | {forced_number} hash value   // #123 or #some.var
    513               {->New expression.numeric(value.expression)}
    514     | {not} bang value                              // !x.y
    515               {->New expression.not(value.expression)}
    516     | {exists} question value      // ?x.y
    517               {->New expression.exists(value.expression)}
    518     | {parens} paren_open expression_list paren_close        // (x.y, a.b, d.e)
    519               {->New expression.sequence([expression_list.expression])}
    520     | {function} [name]:variable paren_open
    521                   expression_list? paren_close // a.b(x, y)
    522               {->New expression.function(
    523                   name.variable,[expression_list.expression])}
    524     ;
    525 
    526   variable {->variable}
    527     = {name}   dollar? word
    528               {->New variable.name(word)}
    529     | {dec_number} dollar dec_number
    530               {->New variable.dec_number(dec_number)}
    531     | {hex_number} dollar hex_number
    532               {->New variable.hex_number(hex_number)}
    533     | {descend_name} variable dot word
    534               {->New variable.descend(
    535                   variable.variable, New variable.name(word))}
    536     | {descend_dec_number} variable dot dec_number
    537               {->New variable.descend(
    538                   variable.variable, New variable.dec_number(dec_number))}
    539     | {descend_hex_number} variable dot hex_number
    540               {->New variable.descend(
    541                   variable.variable, New variable.hex_number(hex_number))}
    542     | {expand} variable bracket_open expression bracket_close
    543               {->New variable.expand(
    544                   variable.variable, expression.expression)}
    545     ;
    546 
    547   number {->expression}
    548     = {unsigned} digits
    549               {->digits.expression}
    550     | {positive} plus digits
    551               {->digits.expression}
    552     | {negative} minus digits
    553               {->New expression.negative(digits.expression)}
    554     ;
    555 
    556   digits {->expression}
    557     = {decimal} dec_number
    558               {->New expression.decimal(dec_number)}
    559     | {hex}     hex_number
    560               {->New expression.hex(hex_number)}
    561     ;
    562 
    563 
    564 /***** Stage 3: The Abstract Syntax Tree
    565  *
    566  * This is the resulting model that will be generated by the parser.
    567  *
    568  * It is represented in Java by a strongly typed node tree (the
    569  * classes are code generated by SableCC). These can be interrogated
    570  * using getter methods, or processed by passing a visitor to the
    571  * tree.
    572  *
    573  * The Abstract Syntax Tree definition below is the only thing
    574  * the Java application need to worry about. All previous definitions
    575  * in this file are taken care of by the generated parser.
    576  *
    577  * Example input:
    578  *   Hello <?cs var:user.name ?>!
    579  *   <?cs if:user.age >= 90 ?>
    580  *   You're way old.
    581  *   <?cs elif:user.age >= 21 ?>
    582  *   You're about the right age.
    583  *   <?cs else ?>
    584  *   You're too young.
    585  *   <?cs /if ?>
    586  *   Seeya!
    587  *
    588  * Results in the tree:
    589  *   AMultipleCommand (start)
    590  *     ADataCommand (command)
    591  *       TData (data) "Hello "
    592  *     AVarCommand (command)
    593  *       AVariableExpression (expression)
    594  *         TWord "user.name" (name)
    595  *     ADataCommand (command)
    596  *       TData "!\n" (data)
    597  *     AIfCommand (command)
    598  *       AGteExpression (expression)
    599  *         AVariableExpression (left)
    600  *           TWord "user.age" (name)
    601  *         ADecimalExpresion (right)
    602  *           TDecNumber "90" (value)
    603  *       ADataCommand (block)
    604  *         TData (data) "\nYou're way old.\n"
    605  *       AGteCommand (otherwise)
    606  *         AEqExpression (expression)
    607  *           AVariableExpression (left)
    608  *             TWord "user.age" (name)
    609  *           ADecimalExpresion (right)
    610  *             TDecNumber "21" (value)
    611  *         ADataCommand (block)
    612  *           TData (data) "\nYou're about the right age.\n"
    613  *         ADataCommand (otherwise)
    614  *           TData (data) "\nYou're too young.\n"
    615  *     ADataCommand (command)
    616  *       TData (data) "\nSeeya!\n"
    617  *
    618  * Although not strictly necessary, tokens are prefixed with 'T.' in
    619  * the grammar so they stand out from the rest of the other rules.
    620  */
    621 Abstract Syntax Tree
    622 
    623   command    = {multiple} command*             // Sequence of commands
    624              | {comment}  position T.comment?  // Contents of <?cs # comment ?>
    625              | {data}     T.data               // Any data outside of <?cs ?>
    626              | {var}      position expression           // var:x statement
    627              | {lvar}     position expression           // lvar:x statement
    628              | {evar}     position expression           // evar:x statement
    629              | {uvar}     position expression           // uvar:x statement
    630              | {set}      position variable expression  // set:x=y statement
    631              | {name}     position variable             // name:x statement
    632              | {escape}   position expression  // escape:x statement
    633                           command              // ... commands in context
    634              | {autoescape}   position expression  // autoescape:x statement
    635                           command              // ... commands in context
    636              | {with}     position variable expression  // with:x=y statement
    637                           command              // ... commands in context
    638              | {loop_to}  position variable    // loop:x=10 statement
    639                           expression           // ... value to end at
    640                           command              // ... commands in loop
    641              | {loop}     position variable    // loop:x=1,10 statement
    642                           [start]:expression   // ... value to start at
    643                           [end]:expression     // ... value to end at
    644                           command              // ... commands in loop
    645              | {loop_inc} position variable    // loop:x=1,10,2 statement
    646                           [start]:expression   // ... value to start at
    647                           [end]:expression     // ... value to end at
    648                           [increment]:expression // . value to increment by
    649                           command              // ... commands in loop
    650              | {each}     position variable expression  // each:x=y statement
    651                           command              // ... commands in loop
    652              | {def}      position [macro]:T.word* // def:some_macro statement
    653                           [arguments]:variable* // ... arguments
    654                           command              // ... commands to execute
    655              | {call}     position [macro]:T.word*  // call:some_macro statement
    656                           [arguments]:expression* // ... arguments
    657              | {if}       position expression  // if:x statement
    658                           [block]:command      // ... commands if true
    659                           [otherwise]:command  // ... commands if false
    660              | {alt}      position expression command   // alt:x statement
    661              | {include}  position expression           // include:x statement
    662              | {hard_include} position expression       // include!x statement
    663              | {linclude} position expression           // linclude:x statement
    664              | {hard_linclude} position expression      // linclude!x statement
    665              | {content_type}  position string          // content-type:x statement
    666              | {inline}   position command              // inline commands
    667              | {noop}                                   // No operation
    668              ;
    669 
    670   position   = {cs_open}  T.cs_open   // We retain the <?cs token in the AST
    671              ;                        // as a convenient way to get the position
    672                                       // of the start of the command.
    673 
    674   expression = {string}      [value]:T.string                     // "hello"
    675              | {numeric}     expression                           // #something
    676              | {decimal}     [value]:T.dec_number                 // 123
    677              | {hex}         [value]:T.hex_number                 // 0x1BF
    678              | {variable}    variable                             // some.thing[1]
    679              | {function}    [name]:variable [args]:expression*   // a.b(x, y)
    680              | {sequence}    [args]:expression*                   // x, y, z
    681              | {negative}    expression                           // -x
    682              | {not}         expression                           // !x
    683              | {exists}      expression                           // ?x
    684              | {comma}       [left]:expression [right]:expression // x, y
    685              | {eq}          [left]:expression [right]:expression // x == y
    686              | {numeric_eq}  [left]:expression [right]:expression // x == y (numeric)
    687              | {ne}          [left]:expression [right]:expression // x != y
    688              | {numeric_ne}  [left]:expression [right]:expression // x != y (numeric)
    689              | {lt}          [left]:expression [right]:expression // x < y
    690              | {gt}          [left]:expression [right]:expression // x > y
    691              | {lte}         [left]:expression [right]:expression // x <= y
    692              | {gte}         [left]:expression [right]:expression // x >= y
    693              | {and}         [left]:expression [right]:expression // x && y
    694              | {or}          [left]:expression [right]:expression // x || y
    695              | {add}         [left]:expression [right]:expression // x + y
    696              | {numeric_add} [left]:expression [right]:expression // x + y (numeric)
    697              | {subtract}    [left]:expression [right]:expression // x - y
    698              | {multiply}    [left]:expression [right]:expression // x * y
    699              | {divide}      [left]:expression [right]:expression // x / y
    700              | {modulo}      [left]:expression [right]:expression // x % y
    701              | {noop}                                             // No operation
    702              ;
    703 
    704   variable   = {name}       T.word                             // something
    705              | {dec_number} T.dec_number                       // 2
    706              | {hex_number} T.hex_number                       // 0xA1
    707              | {descend}  [parent]:variable [child]:variable   // foo.bar
    708              | {expand}   [parent]:variable [child]:expression // foo["bar"]
    709              ;
    710