1 import unittest 2 import textwrap 3 import antlr3 4 import antlr3.tree 5 import stringtemplate3 6 import testbase 7 import sys 8 import os 9 from StringIO import StringIO 10 11 class T(testbase.ANTLRTest): 12 def execParser(self, grammar, grammarEntry, input, group=None): 13 lexerCls, parserCls = self.compileInlineGrammar(grammar) 14 15 cStream = antlr3.StringStream(input) 16 lexer = lexerCls(cStream) 17 tStream = antlr3.CommonTokenStream(lexer) 18 parser = parserCls(tStream) 19 if group is not None: 20 parser.templateLib = group 21 result = getattr(parser, grammarEntry)() 22 if result.st is not None: 23 return result.st.toString() 24 return None 25 26 27 def testInlineTemplate(self): 28 grammar = textwrap.dedent( 29 r'''grammar T; 30 options { 31 language=Python; 32 output=template; 33 } 34 a : ID INT 35 -> template(id={$ID.text}, int={$INT.text}) 36 "id=<id>, int=<int>" 37 ; 38 39 ID : 'a'..'z'+; 40 INT : '0'..'9'+; 41 WS : (' '|'\n') {$channel=HIDDEN;} ; 42 ''' 43 ) 44 45 found = self.execParser( 46 grammar, 'a', 47 "abc 34" 48 ) 49 50 self.failUnlessEqual("id=abc, int=34", found) 51 52 53 def testExternalTemplate(self): 54 templates = textwrap.dedent( 55 '''\ 56 group T; 57 expr(args, op) ::= << 58 [<args; separator={<op>}>] 59 >> 60 ''' 61 ) 62 63 group = stringtemplate3.StringTemplateGroup( 64 file=StringIO(templates), 65 lexer='angle-bracket' 66 ) 67 68 grammar = textwrap.dedent( 69 r'''grammar T2; 70 options { 71 language=Python; 72 output=template; 73 } 74 a : r+=arg OP r+=arg 75 -> expr(op={$OP.text}, args={$r}) 76 ; 77 arg: ID -> template(t={$ID.text}) "<t>"; 78 79 ID : 'a'..'z'+; 80 OP: '+'; 81 WS : (' '|'\n') {$channel=HIDDEN;} ; 82 ''' 83 ) 84 85 found = self.execParser( 86 grammar, 'a', 87 "a + b", 88 group 89 ) 90 91 self.failUnlessEqual("[a+b]", found) 92 93 94 def testEmptyTemplate(self): 95 grammar = textwrap.dedent( 96 r'''grammar T; 97 options { 98 language=Python; 99 output=template; 100 } 101 a : ID INT 102 -> 103 ; 104 105 ID : 'a'..'z'+; 106 INT : '0'..'9'+; 107 WS : (' '|'\n') {$channel=HIDDEN;} ; 108 ''' 109 ) 110 111 found = self.execParser( 112 grammar, 'a', 113 "abc 34" 114 ) 115 116 self.failUnless(found is None) 117 118 119 def testList(self): 120 grammar = textwrap.dedent( 121 r'''grammar T; 122 options { 123 language=Python; 124 output=template; 125 } 126 a: (r+=b)* EOF 127 -> template(r={$r}) 128 "<r; separator=\",\">" 129 ; 130 131 b: ID 132 -> template(t={$ID.text}) "<t>" 133 ; 134 135 ID : 'a'..'z'+; 136 WS : (' '|'\n') {$channel=HIDDEN;} ; 137 ''' 138 ) 139 140 found = self.execParser( 141 grammar, 'a', 142 "abc def ghi" 143 ) 144 145 self.failUnlessEqual("abc,def,ghi", found) 146 147 148 def testAction(self): 149 grammar = textwrap.dedent( 150 r'''grammar T; 151 options { 152 language=Python; 153 output=template; 154 } 155 a: ID 156 -> { stringtemplate3.StringTemplate("hello") } 157 ; 158 159 ID : 'a'..'z'+; 160 WS : (' '|'\n') {$channel=HIDDEN;} ; 161 ''' 162 ) 163 164 found = self.execParser( 165 grammar, 'a', 166 "abc" 167 ) 168 169 self.failUnlessEqual("hello", found) 170 171 172 def testTemplateExpressionInAction(self): 173 grammar = textwrap.dedent( 174 r'''grammar T; 175 options { 176 language=Python; 177 output=template; 178 } 179 a: ID 180 { $st = %{"hello"} } 181 ; 182 183 ID : 'a'..'z'+; 184 WS : (' '|'\n') {$channel=HIDDEN;} ; 185 ''' 186 ) 187 188 found = self.execParser( 189 grammar, 'a', 190 "abc" 191 ) 192 193 self.failUnlessEqual("hello", found) 194 195 196 def testTemplateExpressionInAction2(self): 197 grammar = textwrap.dedent( 198 r'''grammar T; 199 options { 200 language=Python; 201 output=template; 202 } 203 a: ID 204 { 205 res = %{"hello <foo>"} 206 %res.foo = "world"; 207 } 208 -> { res } 209 ; 210 211 ID : 'a'..'z'+; 212 WS : (' '|'\n') {$channel=HIDDEN;} ; 213 ''' 214 ) 215 216 found = self.execParser( 217 grammar, 'a', 218 "abc" 219 ) 220 221 self.failUnlessEqual("hello world", found) 222 223 224 def testIndirectTemplateConstructor(self): 225 templates = textwrap.dedent( 226 '''\ 227 group T; 228 expr(args, op) ::= << 229 [<args; separator={<op>}>] 230 >> 231 ''' 232 ) 233 234 group = stringtemplate3.StringTemplateGroup( 235 file=StringIO(templates), 236 lexer='angle-bracket' 237 ) 238 239 grammar = textwrap.dedent( 240 r'''grammar T; 241 options { 242 language=Python; 243 output=template; 244 } 245 a: ID 246 { 247 $st = %({"expr"})(args={[1, 2, 3]}, op={"+"}) 248 } 249 ; 250 251 ID : 'a'..'z'+; 252 WS : (' '|'\n') {$channel=HIDDEN;} ; 253 ''' 254 ) 255 256 found = self.execParser( 257 grammar, 'a', 258 "abc", 259 group 260 ) 261 262 self.failUnlessEqual("[1+2+3]", found) 263 264 265 def testPredicates(self): 266 grammar = textwrap.dedent( 267 r'''grammar T3; 268 options { 269 language=Python; 270 output=template; 271 } 272 a : ID INT 273 -> {$ID.text=='a'}? template(int={$INT.text}) 274 "A: <int>" 275 -> {$ID.text=='b'}? template(int={$INT.text}) 276 "B: <int>" 277 -> template(int={$INT.text}) 278 "C: <int>" 279 ; 280 281 ID : 'a'..'z'+; 282 INT : '0'..'9'+; 283 WS : (' '|'\n') {$channel=HIDDEN;} ; 284 ''' 285 ) 286 287 found = self.execParser( 288 grammar, 'a', 289 "b 34" 290 ) 291 292 self.failUnlessEqual("B: 34", found) 293 294 295 def testBacktrackingMode(self): 296 grammar = textwrap.dedent( 297 r'''grammar T4; 298 options { 299 language=Python; 300 output=template; 301 backtrack=true; 302 } 303 a : (ID INT)=> ID INT 304 -> template(id={$ID.text}, int={$INT.text}) 305 "id=<id>, int=<int>" 306 ; 307 308 ID : 'a'..'z'+; 309 INT : '0'..'9'+; 310 WS : (' '|'\n') {$channel=HIDDEN;} ; 311 ''' 312 ) 313 314 found = self.execParser( 315 grammar, 'a', 316 "abc 34" 317 ) 318 319 self.failUnlessEqual("id=abc, int=34", found) 320 321 322 def testRewrite(self): 323 grammar = textwrap.dedent( 324 r'''grammar T5; 325 options { 326 language=Python; 327 output=template; 328 rewrite=true; 329 } 330 331 prog: stat+; 332 333 stat 334 : 'if' '(' expr ')' stat 335 | 'return' return_expr ';' 336 | '{' stat* '}' 337 | ID '=' expr ';' 338 ; 339 340 return_expr 341 : expr 342 -> template(t={$text}) <<boom(<t>)>> 343 ; 344 345 expr 346 : ID 347 | INT 348 ; 349 350 ID: 'a'..'z'+; 351 INT: '0'..'9'+; 352 WS: (' '|'\n')+ {$channel=HIDDEN;} ; 353 COMMENT: '/*' (options {greedy=false;} : .)* '*/' {$channel = HIDDEN;} ; 354 ''' 355 ) 356 357 input = textwrap.dedent( 358 '''\ 359 if ( foo ) { 360 b = /* bla */ 2; 361 return 1 /* foo */; 362 } 363 364 /* gnurz */ 365 return 12; 366 ''' 367 ) 368 369 lexerCls, parserCls = self.compileInlineGrammar(grammar) 370 371 cStream = antlr3.StringStream(input) 372 lexer = lexerCls(cStream) 373 tStream = antlr3.TokenRewriteStream(lexer) 374 parser = parserCls(tStream) 375 result = parser.prog() 376 377 found = tStream.toString() 378 379 expected = textwrap.dedent( 380 '''\ 381 if ( foo ) { 382 b = /* bla */ 2; 383 return boom(1) /* foo */; 384 } 385 386 /* gnurz */ 387 return boom(12); 388 ''' 389 ) 390 391 self.failUnlessEqual(expected, found) 392 393 394 def testTreeRewrite(self): 395 grammar = textwrap.dedent( 396 r'''grammar T6; 397 options { 398 language=Python; 399 output=AST; 400 } 401 402 tokens { 403 BLOCK; 404 ASSIGN; 405 } 406 407 prog: stat+; 408 409 stat 410 : IF '(' e=expr ')' s=stat 411 -> ^(IF $e $s) 412 | RETURN expr ';' 413 -> ^(RETURN expr) 414 | '{' stat* '}' 415 -> ^(BLOCK stat*) 416 | ID '=' expr ';' 417 -> ^(ASSIGN ID expr) 418 ; 419 420 expr 421 : ID 422 | INT 423 ; 424 425 IF: 'if'; 426 RETURN: 'return'; 427 ID: 'a'..'z'+; 428 INT: '0'..'9'+; 429 WS: (' '|'\n')+ {$channel=HIDDEN;} ; 430 COMMENT: '/*' (options {greedy=false;} : .)* '*/' {$channel = HIDDEN;} ; 431 ''' 432 ) 433 434 treeGrammar = textwrap.dedent( 435 r'''tree grammar T6Walker; 436 options { 437 language=Python; 438 tokenVocab=T6; 439 ASTLabelType=CommonTree; 440 output=template; 441 rewrite=true; 442 } 443 444 prog: stat+; 445 446 stat 447 : ^(IF expr stat) 448 | ^(RETURN return_expr) 449 | ^(BLOCK stat*) 450 | ^(ASSIGN ID expr) 451 ; 452 453 return_expr 454 : expr 455 -> template(t={$text}) <<boom(<t>)>> 456 ; 457 458 expr 459 : ID 460 | INT 461 ; 462 ''' 463 ) 464 465 input = textwrap.dedent( 466 '''\ 467 if ( foo ) { 468 b = /* bla */ 2; 469 return 1 /* foo */; 470 } 471 472 /* gnurz */ 473 return 12; 474 ''' 475 ) 476 477 lexerCls, parserCls = self.compileInlineGrammar(grammar) 478 walkerCls = self.compileInlineGrammar(treeGrammar) 479 480 cStream = antlr3.StringStream(input) 481 lexer = lexerCls(cStream) 482 tStream = antlr3.TokenRewriteStream(lexer) 483 parser = parserCls(tStream) 484 tree = parser.prog().tree 485 nodes = antlr3.tree.CommonTreeNodeStream(tree) 486 nodes.setTokenStream(tStream) 487 walker = walkerCls(nodes) 488 walker.prog() 489 490 found = tStream.toString() 491 492 expected = textwrap.dedent( 493 '''\ 494 if ( foo ) { 495 b = /* bla */ 2; 496 return boom(1) /* foo */; 497 } 498 499 /* gnurz */ 500 return boom(12); 501 ''' 502 ) 503 504 self.failUnlessEqual(expected, found) 505 506 507 if __name__ == '__main__': 508 unittest.main() 509
