1 # cython: auto_cpdef=True, infer_types=True, language_level=3, py2_import=True 2 # 3 # Parser 4 # 5 6 # This should be done automatically 7 import cython 8 cython.declare(Nodes=object, ExprNodes=object, EncodedString=object, 9 BytesLiteral=object, StringEncoding=object, 10 FileSourceDescriptor=object, lookup_unicodechar=object, 11 Future=object, Options=object, error=object, warning=object, 12 Builtin=object, ModuleNode=object, Utils=object, 13 re=object, _unicode=object, _bytes=object) 14 15 import re 16 from unicodedata import lookup as lookup_unicodechar 17 18 from Cython.Compiler.Scanning import PyrexScanner, FileSourceDescriptor 19 import Nodes 20 import ExprNodes 21 import Builtin 22 import StringEncoding 23 from StringEncoding import EncodedString, BytesLiteral, _unicode, _bytes 24 from ModuleNode import ModuleNode 25 from Errors import error, warning 26 from Cython import Utils 27 import Future 28 import Options 29 30 class Ctx(object): 31 # Parsing context 32 level = 'other' 33 visibility = 'private' 34 cdef_flag = 0 35 typedef_flag = 0 36 api = 0 37 overridable = 0 38 nogil = 0 39 namespace = None 40 templates = None 41 allow_struct_enum_decorator = False 42 43 def __init__(self, **kwds): 44 self.__dict__.update(kwds) 45 46 def __call__(self, **kwds): 47 ctx = Ctx() 48 d = ctx.__dict__ 49 d.update(self.__dict__) 50 d.update(kwds) 51 return ctx 52 53 def p_ident(s, message = "Expected an identifier"): 54 if s.sy == 'IDENT': 55 name = s.systring 56 s.next() 57 return name 58 else: 59 s.error(message) 60 61 def p_ident_list(s): 62 names = [] 63 while s.sy == 'IDENT': 64 names.append(s.systring) 65 s.next() 66 if s.sy != ',': 67 break 68 s.next() 69 return names 70 71 #------------------------------------------ 72 # 73 # Expressions 74 # 75 #------------------------------------------ 76 77 def p_binop_operator(s): 78 pos = s.position() 79 op = s.sy 80 s.next() 81 return op, pos 82 83 def p_binop_expr(s, ops, p_sub_expr): 84 n1 = p_sub_expr(s) 85 while s.sy in ops: 86 op, pos = p_binop_operator(s) 87 n2 = p_sub_expr(s) 88 n1 = ExprNodes.binop_node(pos, op, n1, n2) 89 if op == '/': 90 if Future.division in s.context.future_directives: 91 n1.truedivision = True 92 else: 93 n1.truedivision = None # unknown 94 return n1 95 96 #lambdef: 'lambda' [varargslist] ':' test 97 98 def p_lambdef(s, allow_conditional=True): 99 # s.sy == 'lambda' 100 pos = s.position() 101 s.next() 102 if s.sy == ':': 103 args = [] 104 star_arg = starstar_arg = None 105 else: 106 args, star_arg, starstar_arg = p_varargslist( 107 s, terminator=':', annotated=False) 108 s.expect(':') 109 if allow_conditional: 110 expr = p_test(s) 111 else: 112 expr = p_test_nocond(s) 113 return ExprNodes.LambdaNode( 114 pos, args = args, 115 star_arg = star_arg, starstar_arg = starstar_arg, 116 result_expr = expr) 117 118 #lambdef_nocond: 'lambda' [varargslist] ':' test_nocond 119 120 def p_lambdef_nocond(s): 121 return p_lambdef(s, allow_conditional=False) 122 123 #test: or_test ['if' or_test 'else' test] | lambdef 124 125 def p_test(s): 126 if s.sy == 'lambda': 127 return p_lambdef(s) 128 pos = s.position() 129 expr = p_or_test(s) 130 if s.sy == 'if': 131 s.next() 132 test = p_or_test(s) 133 s.expect('else') 134 other = p_test(s) 135 return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other) 136 else: 137 return expr 138 139 #test_nocond: or_test | lambdef_nocond 140 141 def p_test_nocond(s): 142 if s.sy == 'lambda': 143 return p_lambdef_nocond(s) 144 else: 145 return p_or_test(s) 146 147 #or_test: and_test ('or' and_test)* 148 149 def p_or_test(s): 150 return p_rassoc_binop_expr(s, ('or',), p_and_test) 151 152 def p_rassoc_binop_expr(s, ops, p_subexpr): 153 n1 = p_subexpr(s) 154 if s.sy in ops: 155 pos = s.position() 156 op = s.sy 157 s.next() 158 n2 = p_rassoc_binop_expr(s, ops, p_subexpr) 159 n1 = ExprNodes.binop_node(pos, op, n1, n2) 160 return n1 161 162 #and_test: not_test ('and' not_test)* 163 164 def p_and_test(s): 165 #return p_binop_expr(s, ('and',), p_not_test) 166 return p_rassoc_binop_expr(s, ('and',), p_not_test) 167 168 #not_test: 'not' not_test | comparison 169 170 def p_not_test(s): 171 if s.sy == 'not': 172 pos = s.position() 173 s.next() 174 return ExprNodes.NotNode(pos, operand = p_not_test(s)) 175 else: 176 return p_comparison(s) 177 178 #comparison: expr (comp_op expr)* 179 #comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' 180 181 def p_comparison(s): 182 n1 = p_starred_expr(s) 183 if s.sy in comparison_ops: 184 pos = s.position() 185 op = p_cmp_op(s) 186 n2 = p_starred_expr(s) 187 n1 = ExprNodes.PrimaryCmpNode(pos, 188 operator = op, operand1 = n1, operand2 = n2) 189 if s.sy in comparison_ops: 190 n1.cascade = p_cascaded_cmp(s) 191 return n1 192 193 def p_test_or_starred_expr(s): 194 if s.sy == '*': 195 return p_starred_expr(s) 196 else: 197 return p_test(s) 198 199 def p_starred_expr(s): 200 pos = s.position() 201 if s.sy == '*': 202 starred = True 203 s.next() 204 else: 205 starred = False 206 expr = p_bit_expr(s) 207 if starred: 208 expr = ExprNodes.StarredTargetNode(pos, expr) 209 return expr 210 211 def p_cascaded_cmp(s): 212 pos = s.position() 213 op = p_cmp_op(s) 214 n2 = p_starred_expr(s) 215 result = ExprNodes.CascadedCmpNode(pos, 216 operator = op, operand2 = n2) 217 if s.sy in comparison_ops: 218 result.cascade = p_cascaded_cmp(s) 219 return result 220 221 def p_cmp_op(s): 222 if s.sy == 'not': 223 s.next() 224 s.expect('in') 225 op = 'not_in' 226 elif s.sy == 'is': 227 s.next() 228 if s.sy == 'not': 229 s.next() 230 op = 'is_not' 231 else: 232 op = 'is' 233 else: 234 op = s.sy 235 s.next() 236 if op == '<>': 237 op = '!=' 238 return op 239 240 comparison_ops = cython.declare(set, set([ 241 '<', '>', '==', '>=', '<=', '<>', '!=', 242 'in', 'is', 'not' 243 ])) 244 245 #expr: xor_expr ('|' xor_expr)* 246 247 def p_bit_expr(s): 248 return p_binop_expr(s, ('|',), p_xor_expr) 249 250 #xor_expr: and_expr ('^' and_expr)* 251 252 def p_xor_expr(s): 253 return p_binop_expr(s, ('^',), p_and_expr) 254 255 #and_expr: shift_expr ('&' shift_expr)* 256 257 def p_and_expr(s): 258 return p_binop_expr(s, ('&',), p_shift_expr) 259 260 #shift_expr: arith_expr (('<<'|'>>') arith_expr)* 261 262 def p_shift_expr(s): 263 return p_binop_expr(s, ('<<', '>>'), p_arith_expr) 264 265 #arith_expr: term (('+'|'-') term)* 266 267 def p_arith_expr(s): 268 return p_binop_expr(s, ('+', '-'), p_term) 269 270 #term: factor (('*'|'/'|'%') factor)* 271 272 def p_term(s): 273 return p_binop_expr(s, ('*', '/', '%', '//'), p_factor) 274 275 #factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power 276 277 def p_factor(s): 278 # little indirection for C-ification purposes 279 return _p_factor(s) 280 281 def _p_factor(s): 282 sy = s.sy 283 if sy in ('+', '-', '~'): 284 op = s.sy 285 pos = s.position() 286 s.next() 287 return ExprNodes.unop_node(pos, op, p_factor(s)) 288 elif not s.in_python_file: 289 if sy == '&': 290 pos = s.position() 291 s.next() 292 arg = p_factor(s) 293 return ExprNodes.AmpersandNode(pos, operand = arg) 294 elif sy == "<": 295 return p_typecast(s) 296 elif sy == 'IDENT' and s.systring == "sizeof": 297 return p_sizeof(s) 298 return p_power(s) 299 300 def p_typecast(s): 301 # s.sy == "<" 302 pos = s.position() 303 s.next() 304 base_type = p_c_base_type(s) 305 is_memslice = isinstance(base_type, Nodes.MemoryViewSliceTypeNode) 306 is_template = isinstance(base_type, Nodes.TemplatedTypeNode) 307 is_const = isinstance(base_type, Nodes.CConstTypeNode) 308 if (not is_memslice and not is_template and not is_const 309 and base_type.name is None): 310 s.error("Unknown type") 311 declarator = p_c_declarator(s, empty = 1) 312 if s.sy == '?': 313 s.next() 314 typecheck = 1 315 else: 316 typecheck = 0 317 s.expect(">") 318 operand = p_factor(s) 319 if is_memslice: 320 return ExprNodes.CythonArrayNode(pos, base_type_node=base_type, 321 operand=operand) 322 323 return ExprNodes.TypecastNode(pos, 324 base_type = base_type, 325 declarator = declarator, 326 operand = operand, 327 typecheck = typecheck) 328 329 def p_sizeof(s): 330 # s.sy == ident "sizeof" 331 pos = s.position() 332 s.next() 333 s.expect('(') 334 # Here we decide if we are looking at an expression or type 335 # If it is actually a type, but parsable as an expression, 336 # we treat it as an expression here. 337 if looking_at_expr(s): 338 operand = p_test(s) 339 node = ExprNodes.SizeofVarNode(pos, operand = operand) 340 else: 341 base_type = p_c_base_type(s) 342 declarator = p_c_declarator(s, empty = 1) 343 node = ExprNodes.SizeofTypeNode(pos, 344 base_type = base_type, declarator = declarator) 345 s.expect(')') 346 return node 347 348 def p_yield_expression(s): 349 # s.sy == "yield" 350 pos = s.position() 351 s.next() 352 is_yield_from = False 353 if s.sy == 'from': 354 is_yield_from = True 355 s.next() 356 if s.sy != ')' and s.sy not in statement_terminators: 357 arg = p_testlist(s) 358 else: 359 if is_yield_from: 360 s.error("'yield from' requires a source argument", 361 pos=pos, fatal=False) 362 arg = None 363 if is_yield_from: 364 return ExprNodes.YieldFromExprNode(pos, arg=arg) 365 else: 366 return ExprNodes.YieldExprNode(pos, arg=arg) 367 368 def p_yield_statement(s): 369 # s.sy == "yield" 370 yield_expr = p_yield_expression(s) 371 return Nodes.ExprStatNode(yield_expr.pos, expr=yield_expr) 372 373 #power: atom trailer* ('**' factor)* 374 375 def p_power(s): 376 if s.systring == 'new' and s.peek()[0] == 'IDENT': 377 return p_new_expr(s) 378 n1 = p_atom(s) 379 while s.sy in ('(', '[', '.'): 380 n1 = p_trailer(s, n1) 381 if s.sy == '**': 382 pos = s.position() 383 s.next() 384 n2 = p_factor(s) 385 n1 = ExprNodes.binop_node(pos, '**', n1, n2) 386 return n1 387 388 def p_new_expr(s): 389 # s.systring == 'new'. 390 pos = s.position() 391 s.next() 392 cppclass = p_c_base_type(s) 393 return p_call(s, ExprNodes.NewExprNode(pos, cppclass = cppclass)) 394 395 #trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME 396 397 def p_trailer(s, node1): 398 pos = s.position() 399 if s.sy == '(': 400 return p_call(s, node1) 401 elif s.sy == '[': 402 return p_index(s, node1) 403 else: # s.sy == '.' 404 s.next() 405 name = EncodedString( p_ident(s) ) 406 return ExprNodes.AttributeNode(pos, 407 obj = node1, attribute = name) 408 409 # arglist: argument (',' argument)* [','] 410 # argument: [test '='] test # Really [keyword '='] test 411 412 def p_call_parse_args(s, allow_genexp = True): 413 # s.sy == '(' 414 pos = s.position() 415 s.next() 416 positional_args = [] 417 keyword_args = [] 418 star_arg = None 419 starstar_arg = None 420 while s.sy not in ('**', ')'): 421 if s.sy == '*': 422 if star_arg: 423 s.error("only one star-arg parameter allowed", 424 pos=s.position()) 425 s.next() 426 star_arg = p_test(s) 427 else: 428 arg = p_test(s) 429 if s.sy == '=': 430 s.next() 431 if not arg.is_name: 432 s.error("Expected an identifier before '='", 433 pos=arg.pos) 434 encoded_name = EncodedString(arg.name) 435 keyword = ExprNodes.IdentifierStringNode( 436 arg.pos, value=encoded_name) 437 arg = p_test(s) 438 keyword_args.append((keyword, arg)) 439 else: 440 if keyword_args: 441 s.error("Non-keyword arg following keyword arg", 442 pos=arg.pos) 443 if star_arg: 444 s.error("Non-keyword arg following star-arg", 445 pos=arg.pos) 446 positional_args.append(arg) 447 if s.sy != ',': 448 break 449 s.next() 450 451 if s.sy == 'for': 452 if len(positional_args) == 1 and not star_arg: 453 positional_args = [ p_genexp(s, positional_args[0]) ] 454 elif s.sy == '**': 455 s.next() 456 starstar_arg = p_test(s) 457 if s.sy == ',': 458 s.next() 459 s.expect(')') 460 return positional_args, keyword_args, star_arg, starstar_arg 461 462 def p_call_build_packed_args(pos, positional_args, keyword_args, 463 star_arg, starstar_arg): 464 arg_tuple = None 465 keyword_dict = None 466 if positional_args or not star_arg: 467 arg_tuple = ExprNodes.TupleNode(pos, 468 args = positional_args) 469 if star_arg: 470 star_arg_tuple = ExprNodes.AsTupleNode(pos, arg = star_arg) 471 if arg_tuple: 472 arg_tuple = ExprNodes.binop_node(pos, 473 operator = '+', operand1 = arg_tuple, 474 operand2 = star_arg_tuple) 475 else: 476 arg_tuple = star_arg_tuple 477 if keyword_args or starstar_arg: 478 keyword_args = [ExprNodes.DictItemNode(pos=key.pos, key=key, value=value) 479 for key, value in keyword_args] 480 if starstar_arg: 481 keyword_dict = ExprNodes.KeywordArgsNode( 482 pos, 483 starstar_arg = starstar_arg, 484 keyword_args = keyword_args) 485 else: 486 keyword_dict = ExprNodes.DictNode( 487 pos, key_value_pairs = keyword_args) 488 return arg_tuple, keyword_dict 489 490 def p_call(s, function): 491 # s.sy == '(' 492 pos = s.position() 493 494 positional_args, keyword_args, star_arg, starstar_arg = \ 495 p_call_parse_args(s) 496 497 if not (keyword_args or star_arg or starstar_arg): 498 return ExprNodes.SimpleCallNode(pos, 499 function = function, 500 args = positional_args) 501 else: 502 arg_tuple, keyword_dict = p_call_build_packed_args( 503 pos, positional_args, keyword_args, star_arg, starstar_arg) 504 return ExprNodes.GeneralCallNode(pos, 505 function = function, 506 positional_args = arg_tuple, 507 keyword_args = keyword_dict) 508 509 #lambdef: 'lambda' [varargslist] ':' test 510 511 #subscriptlist: subscript (',' subscript)* [','] 512 513 def p_index(s, base): 514 # s.sy == '[' 515 pos = s.position() 516 s.next() 517 subscripts, is_single_value = p_subscript_list(s) 518 if is_single_value and len(subscripts[0]) == 2: 519 start, stop = subscripts[0] 520 result = ExprNodes.SliceIndexNode(pos, 521 base = base, start = start, stop = stop) 522 else: 523 indexes = make_slice_nodes(pos, subscripts) 524 if is_single_value: 525 index = indexes[0] 526 else: 527 index = ExprNodes.TupleNode(pos, args = indexes) 528 result = ExprNodes.IndexNode(pos, 529 base = base, index = index) 530 s.expect(']') 531 return result 532 533 def p_subscript_list(s): 534 is_single_value = True 535 items = [p_subscript(s)] 536 while s.sy == ',': 537 is_single_value = False 538 s.next() 539 if s.sy == ']': 540 break 541 items.append(p_subscript(s)) 542 return items, is_single_value 543 544 #subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]] 545 546 def p_subscript(s): 547 # Parse a subscript and return a list of 548 # 1, 2 or 3 ExprNodes, depending on how 549 # many slice elements were encountered. 550 pos = s.position() 551 start = p_slice_element(s, (':',)) 552 if s.sy != ':': 553 return [start] 554 s.next() 555 stop = p_slice_element(s, (':', ',', ']')) 556 if s.sy != ':': 557 return [start, stop] 558 s.next() 559 step = p_slice_element(s, (':', ',', ']')) 560 return [start, stop, step] 561 562 def p_slice_element(s, follow_set): 563 # Simple expression which may be missing iff 564 # it is followed by something in follow_set. 565 if s.sy not in follow_set: 566 return p_test(s) 567 else: 568 return None 569 570 def expect_ellipsis(s): 571 s.expect('.') 572 s.expect('.') 573 s.expect('.') 574 575 def make_slice_nodes(pos, subscripts): 576 # Convert a list of subscripts as returned 577 # by p_subscript_list into a list of ExprNodes, 578 # creating SliceNodes for elements with 2 or 579 # more components. 580 result = [] 581 for subscript in subscripts: 582 if len(subscript) == 1: 583 result.append(subscript[0]) 584 else: 585 result.append(make_slice_node(pos, *subscript)) 586 return result 587 588 def make_slice_node(pos, start, stop = None, step = None): 589 if not start: 590 start = ExprNodes.NoneNode(pos) 591 if not stop: 592 stop = ExprNodes.NoneNode(pos) 593 if not step: 594 step = ExprNodes.NoneNode(pos) 595 return ExprNodes.SliceNode(pos, 596 start = start, stop = stop, step = step) 597 598 #atom: '(' [yield_expr|testlist_comp] ')' | '[' [listmaker] ']' | '{' [dict_or_set_maker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+ 599 600 def p_atom(s): 601 pos = s.position() 602 sy = s.sy 603 if sy == '(': 604 s.next() 605 if s.sy == ')': 606 result = ExprNodes.TupleNode(pos, args = []) 607 elif s.sy == 'yield': 608 result = p_yield_expression(s) 609 else: 610 result = p_testlist_comp(s) 611 s.expect(')') 612 return result 613 elif sy == '[': 614 return p_list_maker(s) 615 elif sy == '{': 616 return p_dict_or_set_maker(s) 617 elif sy == '`': 618 return p_backquote_expr(s) 619 elif sy == '.': 620 expect_ellipsis(s) 621 return ExprNodes.EllipsisNode(pos) 622 elif sy == 'INT': 623 return p_int_literal(s) 624 elif sy == 'FLOAT': 625 value = s.systring 626 s.next() 627 return ExprNodes.FloatNode(pos, value = value) 628 elif sy == 'IMAG': 629 value = s.systring[:-1] 630 s.next() 631 return ExprNodes.ImagNode(pos, value = value) 632 elif sy == 'BEGIN_STRING': 633 kind, bytes_value, unicode_value = p_cat_string_literal(s) 634 if kind == 'c': 635 return ExprNodes.CharNode(pos, value = bytes_value) 636 elif kind == 'u': 637 return ExprNodes.UnicodeNode(pos, value = unicode_value, bytes_value = bytes_value) 638 elif kind == 'b': 639 return ExprNodes.BytesNode(pos, value = bytes_value) 640 else: 641 return ExprNodes.StringNode(pos, value = bytes_value, unicode_value = unicode_value) 642 elif sy == 'IDENT': 643 name = EncodedString( s.systring ) 644 s.next() 645 if name == "None": 646 return ExprNodes.NoneNode(pos) 647 elif name == "True": 648 return ExprNodes.BoolNode(pos, value=True) 649 elif name == "False": 650 return ExprNodes.BoolNode(pos, value=False) 651 elif name == "NULL" and not s.in_python_file: 652 return ExprNodes.NullNode(pos) 653 else: 654 return p_name(s, name) 655 else: 656 s.error("Expected an identifier or literal") 657 658 def p_int_literal(s): 659 pos = s.position() 660 value = s.systring 661 s.next() 662 unsigned = "" 663 longness = "" 664 while value[-1] in u"UuLl": 665 if value[-1] in u"Ll": 666 longness += "L" 667 else: 668 unsigned += "U" 669 value = value[:-1] 670 # '3L' is ambiguous in Py2 but not in Py3. '3U' and '3LL' are 671 # illegal in Py2 Python files. All suffixes are illegal in Py3 672 # Python files. 673 is_c_literal = None 674 if unsigned: 675 is_c_literal = True 676 elif longness: 677 if longness == 'LL' or s.context.language_level >= 3: 678 is_c_literal = True 679 if s.in_python_file: 680 if is_c_literal: 681 error(pos, "illegal integer literal syntax in Python source file") 682 is_c_literal = False 683 return ExprNodes.IntNode(pos, 684 is_c_literal = is_c_literal, 685 value = value, 686 unsigned = unsigned, 687 longness = longness) 688 689 690 def p_name(s, name): 691 pos = s.position() 692 if not s.compile_time_expr and name in s.compile_time_env: 693 value = s.compile_time_env.lookup_here(name) 694 node = wrap_compile_time_constant(pos, value) 695 if node is not None: 696 return node 697 return ExprNodes.NameNode(pos, name=name) 698 699 700 def wrap_compile_time_constant(pos, value): 701 rep = repr(value) 702 if value is None: 703 return ExprNodes.NoneNode(pos) 704 elif value is Ellipsis: 705 return ExprNodes.EllipsisNode(pos) 706 elif isinstance(value, bool): 707 return ExprNodes.BoolNode(pos, value=value) 708 elif isinstance(value, int): 709 return ExprNodes.IntNode(pos, value=rep) 710 elif isinstance(value, long): 711 return ExprNodes.IntNode(pos, value=rep, longness="L") 712 elif isinstance(value, float): 713 return ExprNodes.FloatNode(pos, value=rep) 714 elif isinstance(value, _unicode): 715 return ExprNodes.UnicodeNode(pos, value=EncodedString(value)) 716 elif isinstance(value, _bytes): 717 return ExprNodes.BytesNode(pos, value=BytesLiteral(value)) 718 elif isinstance(value, tuple): 719 args = [wrap_compile_time_constant(pos, arg) 720 for arg in value] 721 if None not in args: 722 return ExprNodes.TupleNode(pos, args=args) 723 else: 724 # error already reported 725 return None 726 error(pos, "Invalid type for compile-time constant: %r (type %s)" 727 % (value, value.__class__.__name__)) 728 return None 729 730 731 def p_cat_string_literal(s): 732 # A sequence of one or more adjacent string literals. 733 # Returns (kind, bytes_value, unicode_value) 734 # where kind in ('b', 'c', 'u', '') 735 kind, bytes_value, unicode_value = p_string_literal(s) 736 if kind == 'c' or s.sy != 'BEGIN_STRING': 737 return kind, bytes_value, unicode_value 738 bstrings, ustrings = [bytes_value], [unicode_value] 739 bytes_value = unicode_value = None 740 while s.sy == 'BEGIN_STRING': 741 pos = s.position() 742 next_kind, next_bytes_value, next_unicode_value = p_string_literal(s) 743 if next_kind == 'c': 744 error(pos, "Cannot concatenate char literal with another string or char literal") 745 elif next_kind != kind: 746 error(pos, "Cannot mix string literals of different types, expected %s'', got %s''" % 747 (kind, next_kind)) 748 else: 749 bstrings.append(next_bytes_value) 750 ustrings.append(next_unicode_value) 751 # join and rewrap the partial literals 752 if kind in ('b', 'c', '') or kind == 'u' and None not in bstrings: 753 # Py3 enforced unicode literals are parsed as bytes/unicode combination 754 bytes_value = BytesLiteral( StringEncoding.join_bytes(bstrings) ) 755 bytes_value.encoding = s.source_encoding 756 if kind in ('u', ''): 757 unicode_value = EncodedString( u''.join([ u for u in ustrings if u is not None ]) ) 758 return kind, bytes_value, unicode_value 759 760 def p_opt_string_literal(s, required_type='u'): 761 if s.sy == 'BEGIN_STRING': 762 kind, bytes_value, unicode_value = p_string_literal(s, required_type) 763 if required_type == 'u': 764 return unicode_value 765 elif required_type == 'b': 766 return bytes_value 767 else: 768 s.error("internal parser configuration error") 769 else: 770 return None 771 772 def check_for_non_ascii_characters(string): 773 for c in string: 774 if c >= u'\x80': 775 return True 776 return False 777 778 def p_string_literal(s, kind_override=None): 779 # A single string or char literal. Returns (kind, bvalue, uvalue) 780 # where kind in ('b', 'c', 'u', ''). The 'bvalue' is the source 781 # code byte sequence of the string literal, 'uvalue' is the 782 # decoded Unicode string. Either of the two may be None depending 783 # on the 'kind' of string, only unprefixed strings have both 784 # representations. 785 786 # s.sy == 'BEGIN_STRING' 787 pos = s.position() 788 is_raw = False 789 is_python3_source = s.context.language_level >= 3 790 has_non_ASCII_literal_characters = False 791 kind = s.systring[:1].lower() 792 if kind == 'r': 793 # Py3 allows both 'br' and 'rb' as prefix 794 if s.systring[1:2].lower() == 'b': 795 kind = 'b' 796 else: 797 kind = '' 798 is_raw = True 799 elif kind in 'ub': 800 is_raw = s.systring[1:2].lower() == 'r' 801 elif kind != 'c': 802 kind = '' 803 if kind == '' and kind_override is None and Future.unicode_literals in s.context.future_directives: 804 chars = StringEncoding.StrLiteralBuilder(s.source_encoding) 805 kind = 'u' 806 else: 807 if kind_override is not None and kind_override in 'ub': 808 kind = kind_override 809 if kind == 'u': 810 chars = StringEncoding.UnicodeLiteralBuilder() 811 elif kind == '': 812 chars = StringEncoding.StrLiteralBuilder(s.source_encoding) 813 else: 814 chars = StringEncoding.BytesLiteralBuilder(s.source_encoding) 815 816 while 1: 817 s.next() 818 sy = s.sy 819 systr = s.systring 820 #print "p_string_literal: sy =", sy, repr(s.systring) ### 821 if sy == 'CHARS': 822 chars.append(systr) 823 if is_python3_source and not has_non_ASCII_literal_characters and check_for_non_ascii_characters(systr): 824 has_non_ASCII_literal_characters = True 825 elif sy == 'ESCAPE': 826 if is_raw: 827 chars.append(systr) 828 if is_python3_source and not has_non_ASCII_literal_characters \ 829 and check_for_non_ascii_characters(systr): 830 has_non_ASCII_literal_characters = True 831 else: 832 c = systr[1] 833 if c in u"01234567": 834 chars.append_charval( int(systr[1:], 8) ) 835 elif c in u"'\"\\": 836 chars.append(c) 837 elif c in u"abfnrtv": 838 chars.append( 839 StringEncoding.char_from_escape_sequence(systr)) 840 elif c == u'\n': 841 pass 842 elif c == u'x': # \xXX 843 if len(systr) == 4: 844 chars.append_charval( int(systr[2:], 16) ) 845 else: 846 s.error("Invalid hex escape '%s'" % systr, 847 fatal=False) 848 elif c in u'NUu' and kind in ('u', ''): # \uxxxx, \Uxxxxxxxx, \N{...} 849 chrval = -1 850 if c == u'N': 851 try: 852 chrval = ord(lookup_unicodechar(systr[3:-1])) 853 except KeyError: 854 s.error("Unknown Unicode character name %s" % 855 repr(systr[3:-1]).lstrip('u')) 856 elif len(systr) in (6,10): 857 chrval = int(systr[2:], 16) 858 if chrval > 1114111: # sys.maxunicode: 859 s.error("Invalid unicode escape '%s'" % systr) 860 chrval = -1 861 else: 862 s.error("Invalid unicode escape '%s'" % systr, 863 fatal=False) 864 if chrval >= 0: 865 chars.append_uescape(chrval, systr) 866 else: 867 chars.append(u'\\' + systr[1:]) 868 if is_python3_source and not has_non_ASCII_literal_characters \ 869 and check_for_non_ascii_characters(systr): 870 has_non_ASCII_literal_characters = True 871 elif sy == 'NEWLINE': 872 chars.append(u'\n') 873 elif sy == 'END_STRING': 874 break 875 elif sy == 'EOF': 876 s.error("Unclosed string literal", pos=pos) 877 else: 878 s.error("Unexpected token %r:%r in string literal" % 879 (sy, s.systring)) 880 881 if kind == 'c': 882 unicode_value = None 883 bytes_value = chars.getchar() 884 if len(bytes_value) != 1: 885 error(pos, u"invalid character literal: %r" % bytes_value) 886 else: 887 bytes_value, unicode_value = chars.getstrings() 888 if is_python3_source and has_non_ASCII_literal_characters: 889 # Python 3 forbids literal non-ASCII characters in byte strings 890 if kind != 'u': 891 s.error("bytes can only contain ASCII literal characters.", 892 pos=pos, fatal=False) 893 bytes_value = None 894 s.next() 895 return (kind, bytes_value, unicode_value) 896 897 # list_display ::= "[" [listmaker] "]" 898 # listmaker ::= expression ( comp_for | ( "," expression )* [","] ) 899 # comp_iter ::= comp_for | comp_if 900 # comp_for ::= "for" expression_list "in" testlist [comp_iter] 901 # comp_if ::= "if" test [comp_iter] 902 903 def p_list_maker(s): 904 # s.sy == '[' 905 pos = s.position() 906 s.next() 907 if s.sy == ']': 908 s.expect(']') 909 return ExprNodes.ListNode(pos, args = []) 910 expr = p_test(s) 911 if s.sy == 'for': 912 append = ExprNodes.ComprehensionAppendNode(pos, expr=expr) 913 loop = p_comp_for(s, append) 914 s.expect(']') 915 return ExprNodes.ComprehensionNode( 916 pos, loop=loop, append=append, type = Builtin.list_type, 917 # list comprehensions leak their loop variable in Py2 918 has_local_scope = s.context.language_level >= 3) 919 else: 920 if s.sy == ',': 921 s.next() 922 exprs = p_simple_expr_list(s, expr) 923 else: 924 exprs = [expr] 925 s.expect(']') 926 return ExprNodes.ListNode(pos, args = exprs) 927 928 def p_comp_iter(s, body): 929 if s.sy == 'for': 930 return p_comp_for(s, body) 931 elif s.sy == 'if': 932 return p_comp_if(s, body) 933 else: 934 # insert the 'append' operation into the loop 935 return body 936 937 def p_comp_for(s, body): 938 # s.sy == 'for' 939 pos = s.position() 940 s.next() 941 kw = p_for_bounds(s, allow_testlist=False) 942 kw.update(else_clause = None, body = p_comp_iter(s, body)) 943 return Nodes.ForStatNode(pos, **kw) 944 945 def p_comp_if(s, body): 946 # s.sy == 'if' 947 pos = s.position() 948 s.next() 949 test = p_test_nocond(s) 950 return Nodes.IfStatNode(pos, 951 if_clauses = [Nodes.IfClauseNode(pos, condition = test, 952 body = p_comp_iter(s, body))], 953 else_clause = None ) 954 955 #dictmaker: test ':' test (',' test ':' test)* [','] 956 957 def p_dict_or_set_maker(s): 958 # s.sy == '{' 959 pos = s.position() 960 s.next() 961 if s.sy == '}': 962 s.next() 963 return ExprNodes.DictNode(pos, key_value_pairs = []) 964 item = p_test(s) 965 if s.sy == ',' or s.sy == '}': 966 # set literal 967 values = [item] 968 while s.sy == ',': 969 s.next() 970 if s.sy == '}': 971 break 972 values.append( p_test(s) ) 973 s.expect('}') 974 return ExprNodes.SetNode(pos, args=values) 975 elif s.sy == 'for': 976 # set comprehension 977 append = ExprNodes.ComprehensionAppendNode( 978 item.pos, expr=item) 979 loop = p_comp_for(s, append) 980 s.expect('}') 981 return ExprNodes.ComprehensionNode( 982 pos, loop=loop, append=append, type=Builtin.set_type) 983 elif s.sy == ':': 984 # dict literal or comprehension 985 key = item 986 s.next() 987 value = p_test(s) 988 if s.sy == 'for': 989 # dict comprehension 990 append = ExprNodes.DictComprehensionAppendNode( 991 item.pos, key_expr=key, value_expr=value) 992 loop = p_comp_for(s, append) 993 s.expect('}') 994 return ExprNodes.ComprehensionNode( 995 pos, loop=loop, append=append, type=Builtin.dict_type) 996 else: 997 # dict literal 998 items = [ExprNodes.DictItemNode(key.pos, key=key, value=value)] 999 while s.sy == ',': 1000 s.next() 1001 if s.sy == '}': 1002 break 1003 key = p_test(s) 1004 s.expect(':') 1005 value = p_test(s) 1006 items.append( 1007 ExprNodes.DictItemNode(key.pos, key=key, value=value)) 1008 s.expect('}') 1009 return ExprNodes.DictNode(pos, key_value_pairs=items) 1010 else: 1011 # raise an error 1012 s.expect('}') 1013 return ExprNodes.DictNode(pos, key_value_pairs = []) 1014 1015 # NOTE: no longer in Py3 :) 1016 def p_backquote_expr(s): 1017 # s.sy == '`' 1018 pos = s.position() 1019 s.next() 1020 args = [p_test(s)] 1021 while s.sy == ',': 1022 s.next() 1023 args.append(p_test(s)) 1024 s.expect('`') 1025 if len(args) == 1: 1026 arg = args[0] 1027 else: 1028 arg = ExprNodes.TupleNode(pos, args = args) 1029 return ExprNodes.BackquoteNode(pos, arg = arg) 1030 1031 def p_simple_expr_list(s, expr=None): 1032 exprs = expr is not None and [expr] or [] 1033 while s.sy not in expr_terminators: 1034 exprs.append( p_test(s) ) 1035 if s.sy != ',': 1036 break 1037 s.next() 1038 return exprs 1039 1040 def p_test_or_starred_expr_list(s, expr=None): 1041 exprs = expr is not None and [expr] or [] 1042 while s.sy not in expr_terminators: 1043 exprs.append( p_test_or_starred_expr(s) ) 1044 if s.sy != ',': 1045 break 1046 s.next() 1047 return exprs 1048 1049 1050 #testlist: test (',' test)* [','] 1051 1052 def p_testlist(s): 1053 pos = s.position() 1054 expr = p_test(s) 1055 if s.sy == ',': 1056 s.next() 1057 exprs = p_simple_expr_list(s, expr) 1058 return ExprNodes.TupleNode(pos, args = exprs) 1059 else: 1060 return expr 1061 1062 # testlist_star_expr: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) 1063 1064 def p_testlist_star_expr(s): 1065 pos = s.position() 1066 expr = p_test_or_starred_expr(s) 1067 if s.sy == ',': 1068 s.next() 1069 exprs = p_test_or_starred_expr_list(s, expr) 1070 return ExprNodes.TupleNode(pos, args = exprs) 1071 else: 1072 return expr 1073 1074 # testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) 1075 1076 def p_testlist_comp(s): 1077 pos = s.position() 1078 expr = p_test_or_starred_expr(s) 1079 if s.sy == ',': 1080 s.next() 1081 exprs = p_test_or_starred_expr_list(s, expr) 1082 return ExprNodes.TupleNode(pos, args = exprs) 1083 elif s.sy == 'for': 1084 return p_genexp(s, expr) 1085 else: 1086 return expr 1087 1088 def p_genexp(s, expr): 1089 # s.sy == 'for' 1090 loop = p_comp_for(s, Nodes.ExprStatNode( 1091 expr.pos, expr = ExprNodes.YieldExprNode(expr.pos, arg=expr))) 1092 return ExprNodes.GeneratorExpressionNode(expr.pos, loop=loop) 1093 1094 expr_terminators = cython.declare(set, set([ 1095 ')', ']', '}', ':', '=', 'NEWLINE'])) 1096 1097 #------------------------------------------------------- 1098 # 1099 # Statements 1100 # 1101 #------------------------------------------------------- 1102 1103 def p_global_statement(s): 1104 # assume s.sy == 'global' 1105 pos = s.position() 1106 s.next() 1107 names = p_ident_list(s) 1108 return Nodes.GlobalNode(pos, names = names) 1109 1110 def p_nonlocal_statement(s): 1111 pos = s.position() 1112 s.next() 1113 names = p_ident_list(s) 1114 return Nodes.NonlocalNode(pos, names = names) 1115 1116 def p_expression_or_assignment(s): 1117 expr_list = [p_testlist_star_expr(s)] 1118 if s.sy == '=' and expr_list[0].is_starred: 1119 # This is a common enough error to make when learning Cython to let 1120 # it fail as early as possible and give a very clear error message. 1121 s.error("a starred assignment target must be in a list or tuple" 1122 " - maybe you meant to use an index assignment: var[0] = ...", 1123 pos=expr_list[0].pos) 1124 while s.sy == '=': 1125 s.next() 1126 if s.sy == 'yield': 1127 expr = p_yield_expression(s) 1128 else: 1129 expr = p_testlist_star_expr(s) 1130 expr_list.append(expr) 1131 if len(expr_list) == 1: 1132 if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy): 1133 lhs = expr_list[0] 1134 if isinstance(lhs, ExprNodes.SliceIndexNode): 1135 # implementation requires IndexNode 1136 lhs = ExprNodes.IndexNode( 1137 lhs.pos, 1138 base=lhs.base, 1139 index=make_slice_node(lhs.pos, lhs.start, lhs.stop)) 1140 elif not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode) ): 1141 error(lhs.pos, "Illegal operand for inplace operation.") 1142 operator = s.sy[:-1] 1143 s.next() 1144 if s.sy == 'yield': 1145 rhs = p_yield_expression(s) 1146 else: 1147 rhs = p_testlist(s) 1148 return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs) 1149 expr = expr_list[0] 1150 return Nodes.ExprStatNode(expr.pos, expr=expr) 1151 1152 rhs = expr_list[-1] 1153 if len(expr_list) == 2: 1154 return Nodes.SingleAssignmentNode(rhs.pos, 1155 lhs = expr_list[0], rhs = rhs) 1156 else: 1157 return Nodes.CascadedAssignmentNode(rhs.pos, 1158 lhs_list = expr_list[:-1], rhs = rhs) 1159 1160 def p_print_statement(s): 1161 # s.sy == 'print' 1162 pos = s.position() 1163 ends_with_comma = 0 1164 s.next() 1165 if s.sy == '>>': 1166 s.next() 1167 stream = p_test(s) 1168 if s.sy == ',': 1169 s.next() 1170 ends_with_comma = s.sy in ('NEWLINE', 'EOF') 1171 else: 1172 stream = None 1173 args = [] 1174 if s.sy not in ('NEWLINE', 'EOF'): 1175 args.append(p_test(s)) 1176 while s.sy == ',': 1177 s.next() 1178 if s.sy in ('NEWLINE', 'EOF'): 1179 ends_with_comma = 1 1180 break 1181 args.append(p_test(s)) 1182 arg_tuple = ExprNodes.TupleNode(pos, args = args) 1183 return Nodes.PrintStatNode(pos, 1184 arg_tuple = arg_tuple, stream = stream, 1185 append_newline = not ends_with_comma) 1186 1187 def p_exec_statement(s): 1188 # s.sy == 'exec' 1189 pos = s.position() 1190 s.next() 1191 code = p_bit_expr(s) 1192 if isinstance(code, ExprNodes.TupleNode): 1193 # Py3 compatibility syntax 1194 tuple_variant = True 1195 args = code.args 1196 if len(args) not in (2, 3): 1197 s.error("expected tuple of length 2 or 3, got length %d" % len(args), 1198 pos=pos, fatal=False) 1199 args = [code] 1200 else: 1201 tuple_variant = False 1202 args = [code] 1203 if s.sy == 'in': 1204 if tuple_variant: 1205 s.error("tuple variant of exec does not support additional 'in' arguments", 1206 fatal=False) 1207 s.next() 1208 args.append(p_test(s)) 1209 if s.sy == ',': 1210 s.next() 1211 args.append(p_test(s)) 1212 return Nodes.ExecStatNode(pos, args=args) 1213 1214 def p_del_statement(s): 1215 # s.sy == 'del' 1216 pos = s.position() 1217 s.next() 1218 # FIXME: 'exprlist' in Python 1219 args = p_simple_expr_list(s) 1220 return Nodes.DelStatNode(pos, args = args) 1221 1222 def p_pass_statement(s, with_newline = 0): 1223 pos = s.position() 1224 s.expect('pass') 1225 if with_newline: 1226 s.expect_newline("Expected a newline") 1227 return Nodes.PassStatNode(pos) 1228 1229 def p_break_statement(s): 1230 # s.sy == 'break' 1231 pos = s.position() 1232 s.next() 1233 return Nodes.BreakStatNode(pos) 1234 1235 def p_continue_statement(s): 1236 # s.sy == 'continue' 1237 pos = s.position() 1238 s.next() 1239 return Nodes.ContinueStatNode(pos) 1240 1241 def p_return_statement(s): 1242 # s.sy == 'return' 1243 pos = s.position() 1244 s.next() 1245 if s.sy not in statement_terminators: 1246 value = p_testlist(s) 1247 else: 1248 value = None 1249 return Nodes.ReturnStatNode(pos, value = value) 1250 1251 def p_raise_statement(s): 1252 # s.sy == 'raise' 1253 pos = s.position() 1254 s.next() 1255 exc_type = None 1256 exc_value = None 1257 exc_tb = None 1258 cause = None 1259 if s.sy not in statement_terminators: 1260 exc_type = p_test(s) 1261 if s.sy == ',': 1262 s.next() 1263 exc_value = p_test(s) 1264 if s.sy == ',': 1265 s.next() 1266 exc_tb = p_test(s) 1267 elif s.sy == 'from': 1268 s.next() 1269 cause = p_test(s) 1270 if exc_type or exc_value or exc_tb: 1271 return Nodes.RaiseStatNode(pos, 1272 exc_type = exc_type, 1273 exc_value = exc_value, 1274 exc_tb = exc_tb, 1275 cause = cause) 1276 else: 1277 return Nodes.ReraiseStatNode(pos) 1278 1279 def p_import_statement(s): 1280 # s.sy in ('import', 'cimport') 1281 pos = s.position() 1282 kind = s.sy 1283 s.next() 1284 items = [p_dotted_name(s, as_allowed = 1)] 1285 while s.sy == ',': 1286 s.next() 1287 items.append(p_dotted_name(s, as_allowed = 1)) 1288 stats = [] 1289 for pos, target_name, dotted_name, as_name in items: 1290 dotted_name = EncodedString(dotted_name) 1291 if kind == 'cimport': 1292 stat = Nodes.CImportStatNode(pos, 1293 module_name = dotted_name, 1294 as_name = as_name) 1295 else: 1296 if as_name and "." in dotted_name: 1297 name_list = ExprNodes.ListNode(pos, args = [ 1298 ExprNodes.IdentifierStringNode(pos, value = EncodedString("*"))]) 1299 else: 1300 name_list = None 1301 stat = Nodes.SingleAssignmentNode(pos, 1302 lhs = ExprNodes.NameNode(pos, 1303 name = as_name or target_name), 1304 rhs = ExprNodes.ImportNode(pos, 1305 module_name = ExprNodes.IdentifierStringNode( 1306 pos, value = dotted_name), 1307 level = None, 1308 name_list = name_list)) 1309 stats.append(stat) 1310 return Nodes.StatListNode(pos, stats = stats) 1311 1312 def p_from_import_statement(s, first_statement = 0): 1313 # s.sy == 'from' 1314 pos = s.position() 1315 s.next() 1316 if s.sy == '.': 1317 # count relative import level 1318 level = 0 1319 while s.sy == '.': 1320 level += 1 1321 s.next() 1322 if s.sy == 'cimport': 1323 s.error("Relative cimport is not supported yet") 1324 else: 1325 level = None 1326 if level is not None and s.sy == 'import': 1327 # we are dealing with "from .. import foo, bar" 1328 dotted_name_pos, dotted_name = s.position(), '' 1329 elif level is not None and s.sy == 'cimport': 1330 # "from .. cimport" 1331 s.error("Relative cimport is not supported yet") 1332 else: 1333 (dotted_name_pos, _, dotted_name, _) = \ 1334 p_dotted_name(s, as_allowed = 0) 1335 if s.sy in ('import', 'cimport'): 1336 kind = s.sy 1337 s.next() 1338 else: 1339 s.error("Expected 'import' or 'cimport'") 1340 1341 is_cimport = kind == 'cimport' 1342 is_parenthesized = False 1343 if s.sy == '*': 1344 imported_names = [(s.position(), "*", None, None)] 1345 s.next() 1346 else: 1347 if s.sy == '(': 1348 is_parenthesized = True 1349 s.next() 1350 imported_names = [p_imported_name(s, is_cimport)] 1351 while s.sy == ',': 1352 s.next() 1353 if is_parenthesized and s.sy == ')': 1354 break 1355 imported_names.append(p_imported_name(s, is_cimport)) 1356 if is_parenthesized: 1357 s.expect(')') 1358 dotted_name = EncodedString(dotted_name) 1359 if dotted_name == '__future__': 1360 if not first_statement: 1361 s.error("from __future__ imports must occur at the beginning of the file") 1362 elif level is not None: 1363 s.error("invalid syntax") 1364 else: 1365 for (name_pos, name, as_name, kind) in imported_names: 1366 if name == "braces": 1367 s.error("not a chance", name_pos) 1368 break 1369 try: 1370 directive = getattr(Future, name) 1371 except AttributeError: 1372 s.error("future feature %s is not defined" % name, name_pos) 1373 break 1374 s.context.future_directives.add(directive) 1375 return Nodes.PassStatNode(pos) 1376 elif kind == 'cimport': 1377 return Nodes.FromCImportStatNode(pos, 1378 module_name = dotted_name, 1379 imported_names = imported_names) 1380 else: 1381 imported_name_strings = [] 1382 items = [] 1383 for (name_pos, name, as_name, kind) in imported_names: 1384 encoded_name = EncodedString(name) 1385 imported_name_strings.append( 1386 ExprNodes.IdentifierStringNode(name_pos, value = encoded_name)) 1387 items.append( 1388 (name, 1389 ExprNodes.NameNode(name_pos, 1390 name = as_name or name))) 1391 import_list = ExprNodes.ListNode( 1392 imported_names[0][0], args = imported_name_strings) 1393 dotted_name = EncodedString(dotted_name) 1394 return Nodes.FromImportStatNode(pos, 1395 module = ExprNodes.ImportNode(dotted_name_pos, 1396 module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name), 1397 level = level, 1398 name_list = import_list), 1399 items = items) 1400 1401 imported_name_kinds = cython.declare( 1402 set, set(['class', 'struct', 'union'])) 1403 1404 def p_imported_name(s, is_cimport): 1405 pos = s.position() 1406 kind = None 1407 if is_cimport and s.systring in imported_name_kinds: 1408 kind = s.systring 1409 s.next() 1410 name = p_ident(s) 1411 as_name = p_as_name(s) 1412 return (pos, name, as_name, kind) 1413 1414 def p_dotted_name(s, as_allowed): 1415 pos = s.position() 1416 target_name = p_ident(s) 1417 as_name = None 1418 names = [target_name] 1419 while s.sy == '.': 1420 s.next() 1421 names.append(p_ident(s)) 1422 if as_allowed: 1423 as_name = p_as_name(s) 1424 return (pos, target_name, u'.'.join(names), as_name) 1425 1426 def p_as_name(s): 1427 if s.sy == 'IDENT' and s.systring == 'as': 1428 s.next() 1429 return p_ident(s) 1430 else: 1431 return None 1432 1433 def p_assert_statement(s): 1434 # s.sy == 'assert' 1435 pos = s.position() 1436 s.next() 1437 cond = p_test(s) 1438 if s.sy == ',': 1439 s.next() 1440 value = p_test(s) 1441 else: 1442 value = None 1443 return Nodes.AssertStatNode(pos, cond = cond, value = value) 1444 1445 statement_terminators = cython.declare(set, set([';', 'NEWLINE', 'EOF'])) 1446 1447 def p_if_statement(s): 1448 # s.sy == 'if' 1449 pos = s.position() 1450 s.next() 1451 if_clauses = [p_if_clause(s)] 1452 while s.sy == 'elif': 1453 s.next() 1454 if_clauses.append(p_if_clause(s)) 1455 else_clause = p_else_clause(s) 1456 return Nodes.IfStatNode(pos, 1457 if_clauses = if_clauses, else_clause = else_clause) 1458 1459 def p_if_clause(s): 1460 pos = s.position() 1461 test = p_test(s) 1462 body = p_suite(s) 1463 return Nodes.IfClauseNode(pos, 1464 condition = test, body = body) 1465 1466 def p_else_clause(s): 1467 if s.sy == 'else': 1468 s.next() 1469 return p_suite(s) 1470 else: 1471 return None 1472 1473 def p_while_statement(s): 1474 # s.sy == 'while' 1475 pos = s.position() 1476 s.next() 1477 test = p_test(s) 1478 body = p_suite(s) 1479 else_clause = p_else_clause(s) 1480 return Nodes.WhileStatNode(pos, 1481 condition = test, body = body, 1482 else_clause = else_clause) 1483 1484 def p_for_statement(s): 1485 # s.sy == 'for' 1486 pos = s.position() 1487 s.next() 1488 kw = p_for_bounds(s, allow_testlist=True) 1489 body = p_suite(s) 1490 else_clause = p_else_clause(s) 1491 kw.update(body = body, else_clause = else_clause) 1492 return Nodes.ForStatNode(pos, **kw) 1493 1494 def p_for_bounds(s, allow_testlist=True): 1495 target = p_for_target(s) 1496 if s.sy == 'in': 1497 s.next() 1498 iterator = p_for_iterator(s, allow_testlist) 1499 return dict( target = target, iterator = iterator ) 1500 elif not s.in_python_file: 1501 if s.sy == 'from': 1502 s.next() 1503 bound1 = p_bit_expr(s) 1504 else: 1505 # Support shorter "for a <= x < b" syntax 1506 bound1, target = target, None 1507 rel1 = p_for_from_relation(s) 1508 name2_pos = s.position() 1509 name2 = p_ident(s) 1510 rel2_pos = s.position() 1511 rel2 = p_for_from_relation(s) 1512 bound2 = p_bit_expr(s) 1513 step = p_for_from_step(s) 1514 if target is None: 1515 target = ExprNodes.NameNode(name2_pos, name = name2) 1516 else: 1517 if not target.is_name: 1518 error(target.pos, 1519 "Target of for-from statement must be a variable name") 1520 elif name2 != target.name: 1521 error(name2_pos, 1522 "Variable name in for-from range does not match target") 1523 if rel1[0] != rel2[0]: 1524 error(rel2_pos, 1525 "Relation directions in for-from do not match") 1526 return dict(target = target, 1527 bound1 = bound1, 1528 relation1 = rel1, 1529 relation2 = rel2, 1530 bound2 = bound2, 1531 step = step, 1532 ) 1533 else: 1534 s.expect('in') 1535 return {} 1536 1537 def p_for_from_relation(s): 1538 if s.sy in inequality_relations: 1539 op = s.sy 1540 s.next() 1541 return op 1542 else: 1543 s.error("Expected one of '<', '<=', '>' '>='") 1544 1545 def p_for_from_step(s): 1546 if s.sy == 'IDENT' and s.systring == 'by': 1547 s.next() 1548 step = p_bit_expr(s) 1549 return step 1550 else: 1551 return None 1552 1553 inequality_relations = cython.declare(set, set(['<', '<=', '>', '>='])) 1554 1555 def p_target(s, terminator): 1556 pos = s.position() 1557 expr = p_starred_expr(s) 1558 if s.sy == ',': 1559 s.next() 1560 exprs = [expr] 1561 while s.sy != terminator: 1562 exprs.append(p_starred_expr(s)) 1563 if s.sy != ',': 1564 break 1565 s.next() 1566 return ExprNodes.TupleNode(pos, args = exprs) 1567 else: 1568 return expr 1569 1570 def p_for_target(s): 1571 return p_target(s, 'in') 1572 1573 def p_for_iterator(s, allow_testlist=True): 1574 pos = s.position() 1575 if allow_testlist: 1576 expr = p_testlist(s) 1577 else: 1578 expr = p_or_test(s) 1579 return ExprNodes.IteratorNode(pos, sequence = expr) 1580 1581 def p_try_statement(s): 1582 # s.sy == 'try' 1583 pos = s.position() 1584 s.next() 1585 body = p_suite(s) 1586 except_clauses = [] 1587 else_clause = None 1588 if s.sy in ('except', 'else'): 1589 while s.sy == 'except': 1590 except_clauses.append(p_except_clause(s)) 1591 if s.sy == 'else': 1592 s.next() 1593 else_clause = p_suite(s) 1594 body = Nodes.TryExceptStatNode(pos, 1595 body = body, except_clauses = except_clauses, 1596 else_clause = else_clause) 1597 if s.sy != 'finally': 1598 return body 1599 # try-except-finally is equivalent to nested try-except/try-finally 1600 if s.sy == 'finally': 1601 s.next() 1602 finally_clause = p_suite(s) 1603 return Nodes.TryFinallyStatNode(pos, 1604 body = body, finally_clause = finally_clause) 1605 else: 1606 s.error("Expected 'except' or 'finally'") 1607 1608 def p_except_clause(s): 1609 # s.sy == 'except' 1610 pos = s.position() 1611 s.next() 1612 exc_type = None 1613 exc_value = None 1614 is_except_as = False 1615 if s.sy != ':': 1616 exc_type = p_test(s) 1617 # normalise into list of single exception tests 1618 if isinstance(exc_type, ExprNodes.TupleNode): 1619 exc_type = exc_type.args 1620 else: 1621 exc_type = [exc_type] 1622 if s.sy == ',' or (s.sy == 'IDENT' and s.systring == 'as' 1623 and s.context.language_level == 2): 1624 s.next() 1625 exc_value = p_test(s) 1626 elif s.sy == 'IDENT' and s.systring == 'as': 1627 # Py3 syntax requires a name here 1628 s.next() 1629 pos2 = s.position() 1630 name = p_ident(s) 1631 exc_value = ExprNodes.NameNode(pos2, name = name) 1632 is_except_as = True 1633 body = p_suite(s) 1634 return Nodes.ExceptClauseNode(pos, 1635 pattern = exc_type, target = exc_value, 1636 body = body, is_except_as=is_except_as) 1637 1638 def p_include_statement(s, ctx): 1639 pos = s.position() 1640 s.next() # 'include' 1641 unicode_include_file_name = p_string_literal(s, 'u')[2] 1642 s.expect_newline("Syntax error in include statement") 1643 if s.compile_time_eval: 1644 include_file_name = unicode_include_file_name 1645 include_file_path = s.context.find_include_file(include_file_name, pos) 1646 if include_file_path: 1647 s.included_files.append(include_file_name) 1648 f = Utils.open_source_file(include_file_path, mode="rU") 1649 source_desc = FileSourceDescriptor(include_file_path) 1650 s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments) 1651 try: 1652 tree = p_statement_list(s2, ctx) 1653 finally: 1654 f.close() 1655 return tree 1656 else: 1657 return None 1658 else: 1659 return Nodes.PassStatNode(pos) 1660 1661 def p_with_statement(s): 1662 s.next() # 'with' 1663 if s.systring == 'template' and not s.in_python_file: 1664 node = p_with_template(s) 1665 else: 1666 node = p_with_items(s) 1667 return node 1668 1669 def p_with_items(s): 1670 pos = s.position() 1671 if not s.in_python_file and s.sy == 'IDENT' and s.systring in ('nogil', 'gil'): 1672 state = s.systring 1673 s.next() 1674 if s.sy == ',': 1675 s.next() 1676 body = p_with_items(s) 1677 else: 1678 body = p_suite(s) 1679 return Nodes.GILStatNode(pos, state = state, body = body) 1680 else: 1681 manager = p_test(s) 1682 target = None 1683 if s.sy == 'IDENT' and s.systring == 'as': 1684 s.next() 1685 target = p_starred_expr(s) 1686 if s.sy == ',': 1687 s.next() 1688 body = p_with_items(s) 1689 else: 1690 body = p_suite(s) 1691 return Nodes.WithStatNode(pos, manager = manager, 1692 target = target, body = body) 1693 1694 def p_with_template(s): 1695 pos = s.position() 1696 templates = [] 1697 s.next() 1698 s.expect('[') 1699 templates.append(s.systring) 1700 s.next() 1701 while s.systring == ',': 1702 s.next() 1703 templates.append(s.systring) 1704 s.next() 1705 s.expect(']') 1706 if s.sy == ':': 1707 s.next() 1708 s.expect_newline("Syntax error in template function declaration") 1709 s.expect_indent() 1710 body_ctx = Ctx() 1711 body_ctx.templates = templates 1712 func_or_var = p_c_func_or_var_declaration(s, pos, body_ctx) 1713 s.expect_dedent() 1714 return func_or_var 1715 else: 1716 error(pos, "Syntax error in template function declaration") 1717 1718 def p_simple_statement(s, first_statement = 0): 1719 #print "p_simple_statement:", s.sy, s.systring ### 1720 if s.sy == 'global': 1721 node = p_global_statement(s) 1722 elif s.sy == 'nonlocal': 1723 node = p_nonlocal_statement(s) 1724 elif s.sy == 'print': 1725 node = p_print_statement(s) 1726 elif s.sy == 'exec': 1727 node = p_exec_statement(s) 1728 elif s.sy == 'del': 1729 node = p_del_statement(s) 1730 elif s.sy == 'break': 1731 node = p_break_statement(s) 1732 elif s.sy == 'continue': 1733 node = p_continue_statement(s) 1734 elif s.sy == 'return': 1735 node = p_return_statement(s) 1736 elif s.sy == 'raise': 1737 node = p_raise_statement(s) 1738 elif s.sy in ('import', 'cimport'): 1739 node = p_import_statement(s) 1740 elif s.sy == 'from': 1741 node = p_from_import_statement(s, first_statement = first_statement) 1742 elif s.sy == 'yield': 1743 node = p_yield_statement(s) 1744 elif s.sy == 'assert': 1745 node = p_assert_statement(s) 1746 elif s.sy == 'pass': 1747 node = p_pass_statement(s) 1748 else: 1749 node = p_expression_or_assignment(s) 1750 return node 1751 1752 def p_simple_statement_list(s, ctx, first_statement = 0): 1753 # Parse a series of simple statements on one line 1754 # separated by semicolons. 1755 stat = p_simple_statement(s, first_statement = first_statement) 1756 pos = stat.pos 1757 stats = [] 1758 if not isinstance(stat, Nodes.PassStatNode): 1759 stats.append(stat) 1760 while s.sy == ';': 1761 #print "p_simple_statement_list: maybe more to follow" ### 1762 s.next() 1763 if s.sy in ('NEWLINE', 'EOF'): 1764 break 1765 stat = p_simple_statement(s, first_statement = first_statement) 1766 if isinstance(stat, Nodes.PassStatNode): 1767 continue 1768 stats.append(stat) 1769 first_statement = False 1770 1771 if not stats: 1772 stat = Nodes.PassStatNode(pos) 1773 elif len(stats) == 1: 1774 stat = stats[0] 1775 else: 1776 stat = Nodes.StatListNode(pos, stats = stats) 1777 s.expect_newline("Syntax error in simple statement list") 1778 return stat 1779 1780 def p_compile_time_expr(s): 1781 old = s.compile_time_expr 1782 s.compile_time_expr = 1 1783 expr = p_testlist(s) 1784 s.compile_time_expr = old 1785 return expr 1786 1787 def p_DEF_statement(s): 1788 pos = s.position() 1789 denv = s.compile_time_env 1790 s.next() # 'DEF' 1791 name = p_ident(s) 1792 s.expect('=') 1793 expr = p_compile_time_expr(s) 1794 value = expr.compile_time_value(denv) 1795 #print "p_DEF_statement: %s = %r" % (name, value) ### 1796 denv.declare(name, value) 1797 s.expect_newline() 1798 return Nodes.PassStatNode(pos) 1799 1800 def p_IF_statement(s, ctx): 1801 pos = s.position() 1802 saved_eval = s.compile_time_eval 1803 current_eval = saved_eval 1804 denv = s.compile_time_env 1805 result = None 1806 while 1: 1807 s.next() # 'IF' or 'ELIF' 1808 expr = p_compile_time_expr(s) 1809 s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv)) 1810 body = p_suite(s, ctx) 1811 if s.compile_time_eval: 1812 result = body 1813 current_eval = 0 1814 if s.sy != 'ELIF': 1815 break 1816 if s.sy == 'ELSE': 1817 s.next() 1818 s.compile_time_eval = current_eval 1819 body = p_suite(s, ctx) 1820 if current_eval: 1821 result = body 1822 if not result: 1823 result = Nodes.PassStatNode(pos) 1824 s.compile_time_eval = saved_eval 1825 return result 1826 1827 def p_statement(s, ctx, first_statement = 0): 1828 cdef_flag = ctx.cdef_flag 1829 decorators = None 1830 if s.sy == 'ctypedef': 1831 if ctx.level not in ('module', 'module_pxd'): 1832 s.error("ctypedef statement not allowed here") 1833 #if ctx.api: 1834 # error(s.position(), "'api' not allowed with 'ctypedef'") 1835 return p_ctypedef_statement(s, ctx) 1836 elif s.sy == 'DEF': 1837 return p_DEF_statement(s) 1838 elif s.sy == 'IF': 1839 return p_IF_statement(s, ctx) 1840 elif s.sy == 'DECORATOR': 1841 if ctx.level not in ('module', 'class', 'c_class', 'function', 'property', 'module_pxd', 'c_class_pxd', 'other'): 1842 s.error('decorator not allowed here') 1843 s.level = ctx.level 1844 decorators = p_decorators(s) 1845 bad_toks = 'def', 'cdef', 'cpdef', 'class' 1846 if not ctx.allow_struct_enum_decorator and s.sy not in bad_toks: 1847 s.error("Decorators can only be followed by functions or classes") 1848 elif s.sy == 'pass' and cdef_flag: 1849 # empty cdef block 1850 return p_pass_statement(s, with_newline = 1) 1851 1852 overridable = 0 1853 if s.sy == 'cdef': 1854 cdef_flag = 1 1855 s.next() 1856 elif s.sy == 'cpdef': 1857 cdef_flag = 1 1858 overridable = 1 1859 s.next() 1860 if cdef_flag: 1861 if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'): 1862 s.error('cdef statement not allowed here') 1863 s.level = ctx.level 1864 node = p_cdef_statement(s, ctx(overridable = overridable)) 1865 if decorators is not None: 1866 tup = Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode 1867 if ctx.allow_struct_enum_decorator: 1868 tup += Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode 1869 if not isinstance(node, tup): 1870 s.error("Decorators can only be followed by functions or classes") 1871 node.decorators = decorators 1872 return node 1873 else: 1874 if ctx.api: 1875 s.error("'api' not allowed with this statement", fatal=False) 1876 elif s.sy == 'def': 1877 # def statements aren't allowed in pxd files, except 1878 # as part of a cdef class 1879 if ('pxd' in ctx.level) and (ctx.level != 'c_class_pxd'): 1880 s.error('def statement not allowed here') 1881 s.level = ctx.level 1882 return p_def_statement(s, decorators) 1883 elif s.sy == 'class': 1884 if ctx.level not in ('module', 'function', 'class', 'other'): 1885 s.error("class definition not allowed here") 1886 return p_class_statement(s, decorators) 1887 elif s.sy == 'include': 1888 if ctx.level not in ('module', 'module_pxd'): 1889 s.error("include statement not allowed here") 1890 return p_include_statement(s, ctx) 1891 elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property': 1892 return p_property_decl(s) 1893 elif s.sy == 'pass' and ctx.level != 'property': 1894 return p_pass_statement(s, with_newline=True) 1895 else: 1896 if ctx.level in ('c_class_pxd', 'property'): 1897 node = p_ignorable_statement(s) 1898 if node is not None: 1899 return node 1900 s.error("Executable statement not allowed here") 1901 if s.sy == 'if': 1902 return p_if_statement(s) 1903 elif s.sy == 'while': 1904 return p_while_statement(s) 1905 elif s.sy == 'for': 1906 return p_for_statement(s) 1907 elif s.sy == 'try': 1908 return p_try_statement(s) 1909 elif s.sy == 'with': 1910 return p_with_statement(s) 1911 else: 1912 return p_simple_statement_list( 1913 s, ctx, first_statement = first_statement) 1914 1915 def p_statement_list(s, ctx, first_statement = 0): 1916 # Parse a series of statements separated by newlines. 1917 pos = s.position() 1918 stats = [] 1919 while s.sy not in ('DEDENT', 'EOF'): 1920 stat = p_statement(s, ctx, first_statement = first_statement) 1921 if isinstance(stat, Nodes.PassStatNode): 1922 continue 1923 stats.append(stat) 1924 first_statement = False 1925 if not stats: 1926 return Nodes.PassStatNode(pos) 1927 elif len(stats) == 1: 1928 return stats[0] 1929 else: 1930 return Nodes.StatListNode(pos, stats = stats) 1931 1932 1933 def p_suite(s, ctx=Ctx()): 1934 return p_suite_with_docstring(s, ctx, with_doc_only=False)[1] 1935 1936 1937 def p_suite_with_docstring(s, ctx, with_doc_only=False): 1938 s.expect(':') 1939 doc = None 1940 if s.sy == 'NEWLINE': 1941 s.next() 1942 s.expect_indent() 1943 if with_doc_only: 1944 doc = p_doc_string(s) 1945 body = p_statement_list(s, ctx) 1946 s.expect_dedent() 1947 else: 1948 if ctx.api: 1949 s.error("'api' not allowed with this statement", fatal=False) 1950 if ctx.level in ('module', 'class', 'function', 'other'): 1951 body = p_simple_statement_list(s, ctx) 1952 else: 1953 body = p_pass_statement(s) 1954 s.expect_newline("Syntax error in declarations") 1955 if not with_doc_only: 1956 doc, body = _extract_docstring(body) 1957 return doc, body 1958 1959 1960 def p_positional_and_keyword_args(s, end_sy_set, templates = None): 1961 """ 1962 Parses positional and keyword arguments. end_sy_set 1963 should contain any s.sy that terminate the argument list. 1964 Argument expansion (* and **) are not allowed. 1965 1966 Returns: (positional_args, keyword_args) 1967 """ 1968 positional_args = [] 1969 keyword_args = [] 1970 pos_idx = 0 1971 1972 while s.sy not in end_sy_set: 1973 if s.sy == '*' or s.sy == '**': 1974 s.error('Argument expansion not allowed here.', fatal=False) 1975 1976 parsed_type = False 1977 if s.sy == 'IDENT' and s.peek()[0] == '=': 1978 ident = s.systring 1979 s.next() # s.sy is '=' 1980 s.next() 1981 if looking_at_expr(s): 1982 arg = p_test(s) 1983 else: 1984 base_type = p_c_base_type(s, templates = templates) 1985 declarator = p_c_declarator(s, empty = 1) 1986 arg = Nodes.CComplexBaseTypeNode(base_type.pos, 1987 base_type = base_type, declarator = declarator) 1988 parsed_type = True 1989 keyword_node = ExprNodes.IdentifierStringNode( 1990 arg.pos, value = EncodedString(ident)) 1991 keyword_args.append((keyword_node, arg)) 1992 was_keyword = True 1993 1994 else: 1995 if looking_at_expr(s): 1996 arg = p_test(s) 1997 else: 1998 base_type = p_c_base_type(s, templates = templates) 1999 declarator = p_c_declarator(s, empty = 1) 2000 arg = Nodes.CComplexBaseTypeNode(base_type.pos, 2001 base_type = base_type, declarator = declarator) 2002 parsed_type = True 2003 positional_args.append(arg) 2004 pos_idx += 1 2005 if len(keyword_args) > 0: 2006 s.error("Non-keyword arg following keyword arg", 2007 pos=arg.pos) 2008 2009 if s.sy != ',': 2010 if s.sy not in end_sy_set: 2011 if parsed_type: 2012 s.error("Unmatched %s" % " or ".join(end_sy_set)) 2013 break 2014 s.next() 2015 return positional_args, keyword_args 2016 2017 def p_c_base_type(s, self_flag = 0, nonempty = 0, templates = None): 2018 # If self_flag is true, this is the base type for the 2019 # self argument of a C method of an extension type. 2020 if s.sy == '(': 2021 return p_c_complex_base_type(s, templates = templates) 2022 else: 2023 return p_c_simple_base_type(s, self_flag, nonempty = nonempty, templates = templates) 2024 2025 def p_calling_convention(s): 2026 if s.sy == 'IDENT' and s.systring in calling_convention_words: 2027 result = s.systring 2028 s.next() 2029 return result 2030 else: 2031 return "" 2032 2033 calling_convention_words = cython.declare( 2034 set, set(["__stdcall", "__cdecl", "__fastcall"])) 2035 2036 def p_c_complex_base_type(s, templates = None): 2037 # s.sy == '(' 2038 pos = s.position() 2039 s.next() 2040 base_type = p_c_base_type(s, templates = templates) 2041 declarator = p_c_declarator(s, empty = 1) 2042 s.expect(')') 2043 type_node = Nodes.CComplexBaseTypeNode(pos, 2044 base_type = base_type, declarator = declarator) 2045 if s.sy == '[': 2046 if is_memoryviewslice_access(s): 2047 type_node = p_memoryviewslice_access(s, type_node) 2048 else: 2049 type_node = p_buffer_or_template(s, type_node, templates) 2050 return type_node 2051 2052 2053 def p_c_simple_base_type(s, self_flag, nonempty, templates = None): 2054 #print "p_c_simple_base_type: self_flag =", self_flag, nonempty 2055 is_basic = 0 2056 signed = 1 2057 longness = 0 2058 complex = 0 2059 module_path = [] 2060 pos = s.position() 2061 if not s.sy == 'IDENT': 2062 error(pos, "Expected an identifier, found '%s'" % s.sy) 2063 if s.systring == 'const': 2064 s.next() 2065 base_type = p_c_base_type(s, 2066 self_flag = self_flag, nonempty = nonempty, templates = templates) 2067 return Nodes.CConstTypeNode(pos, base_type = base_type) 2068 if looking_at_base_type(s): 2069 #print "p_c_simple_base_type: looking_at_base_type at", s.position() 2070 is_basic = 1 2071 if s.sy == 'IDENT' and s.systring in special_basic_c_types: 2072 signed, longness = special_basic_c_types[s.systring] 2073 name = s.systring 2074 s.next() 2075 else: 2076 signed, longness = p_sign_and_longness(s) 2077 if s.sy == 'IDENT' and s.systring in basic_c_type_names: 2078 name = s.systring 2079 s.next() 2080 else: 2081 name = 'int' # long [int], short [int], long [int] complex, etc. 2082 if s.sy == 'IDENT' and s.systring == 'complex': 2083 complex = 1 2084 s.next() 2085 elif looking_at_dotted_name(s): 2086 #print "p_c_simple_base_type: looking_at_type_name at", s.position() 2087 name = s.systring 2088 s.next() 2089 while s.sy == '.': 2090 module_path.append(name) 2091 s.next() 2092 name = p_ident(s) 2093 else: 2094 name = s.systring 2095 s.next() 2096 if nonempty and s.sy != 'IDENT': 2097 # Make sure this is not a declaration of a variable or function. 2098 if s.sy == '(': 2099 s.next() 2100 if (s.sy == '*' or s.sy == '**' or s.sy == '&' 2101 or (s.sy == 'IDENT' and s.systring in calling_convention_words)): 2102 s.put_back('(', '(') 2103 else: 2104 s.put_back('(', '(') 2105 s.put_back('IDENT', name) 2106 name = None 2107 elif s.sy not in ('*', '**', '[', '&'): 2108 s.put_back('IDENT', name) 2109 name = None 2110 2111 type_node = Nodes.CSimpleBaseTypeNode(pos, 2112 name = name, module_path = module_path, 2113 is_basic_c_type = is_basic, signed = signed, 2114 complex = complex, longness = longness, 2115 is_self_arg = self_flag, templates = templates) 2116 2117 # declarations here. 2118 if s.sy == '[': 2119 if is_memoryviewslice_access(s): 2120 type_node = p_memoryviewslice_access(s, type_node) 2121 else: 2122 type_node = p_buffer_or_template(s, type_node, templates) 2123 2124 if s.sy == '.': 2125 s.next() 2126 name = p_ident(s) 2127 type_node = Nodes.CNestedBaseTypeNode(pos, base_type = type_node, name = name) 2128 2129 return type_node 2130 2131 def p_buffer_or_template(s, base_type_node, templates): 2132 # s.sy == '[' 2133 pos = s.position() 2134 s.next() 2135 # Note that buffer_positional_options_count=1, so the only positional argument is dtype. 2136 # For templated types, all parameters are types. 2137 positional_args, keyword_args = ( 2138 p_positional_and_keyword_args(s, (']',), templates) 2139 ) 2140 s.expect(']') 2141 2142 if s.sy == '[': 2143 base_type_node = p_buffer_or_template(s, base_type_node, templates) 2144 2145 keyword_dict = ExprNodes.DictNode(pos, 2146 key_value_pairs = [ 2147 ExprNodes.DictItemNode(pos=key.pos, key=key, value=value) 2148 for key, value in keyword_args 2149 ]) 2150 result = Nodes.TemplatedTypeNode(pos, 2151 positional_args = positional_args, 2152 keyword_args = keyword_dict, 2153 base_type_node = base_type_node) 2154 return result 2155 2156 def p_bracketed_base_type(s, base_type_node, nonempty, empty): 2157 # s.sy == '[' 2158 if empty and not nonempty: 2159 # sizeof-like thing. Only anonymous C arrays allowed (int[SIZE]). 2160 return base_type_node 2161 elif not empty and nonempty: 2162 # declaration of either memoryview slice or buffer. 2163 if is_memoryviewslice_access(s): 2164 return p_memoryviewslice_access(s, base_type_node) 2165 else: 2166 return p_buffer_or_template(s, base_type_node, None) 2167 # return p_buffer_access(s, base_type_node) 2168 elif not empty and not nonempty: 2169 # only anonymous C arrays and memoryview slice arrays here. We 2170 # disallow buffer declarations for now, due to ambiguity with anonymous 2171 # C arrays. 2172 if is_memoryviewslice_access(s): 2173 return p_memoryviewslice_access(s, base_type_node) 2174 else: 2175 return base_type_node 2176 2177 def is_memoryviewslice_access(s): 2178 # s.sy == '[' 2179 # a memoryview slice declaration is distinguishable from a buffer access 2180 # declaration by the first entry in the bracketed list. The buffer will 2181 # not have an unnested colon in the first entry; the memoryview slice will. 2182 saved = [(s.sy, s.systring)] 2183 s.next() 2184 retval = False 2185 if s.systring == ':': 2186 retval = True 2187 elif s.sy == 'INT': 2188 saved.append((s.sy, s.systring)) 2189 s.next() 2190 if s.sy == ':': 2191 retval = True 2192 2193 for sv in saved[::-1]: 2194 s.put_back(*sv) 2195 2196 return retval 2197 2198 def p_memoryviewslice_access(s, base_type_node): 2199 # s.sy == '[' 2200 pos = s.position() 2201 s.next() 2202 subscripts, _ = p_subscript_list(s) 2203 # make sure each entry in subscripts is a slice 2204 for subscript in subscripts: 2205 if len(subscript) < 2: 2206 s.error("An axis specification in memoryview declaration does not have a ':'.") 2207 s.expect(']') 2208 indexes = make_slice_nodes(pos, subscripts) 2209 result = Nodes.MemoryViewSliceTypeNode(pos, 2210 base_type_node = base_type_node, 2211 axes = indexes) 2212 return result 2213 2214 def looking_at_name(s): 2215 return s.sy == 'IDENT' and not s.systring in calling_convention_words 2216 2217 def looking_at_expr(s): 2218 if s.systring in base_type_start_words: 2219 return False 2220 elif s.sy == 'IDENT': 2221 is_type = False 2222 name = s.systring 2223 dotted_path = [] 2224 s.next() 2225 2226 while s.sy == '.': 2227 s.next() 2228 dotted_path.append(s.systring) 2229 s.expect('IDENT') 2230 2231 saved = s.sy, s.systring 2232 if s.sy == 'IDENT': 2233 is_type = True 2234 elif s.sy == '*' or s.sy == '**': 2235 s.next() 2236 is_type = s.sy in (')', ']') 2237 s.put_back(*saved) 2238 elif s.sy == '(': 2239 s.next() 2240 is_type = s.sy == '*' 2241 s.put_back(*saved) 2242 elif s.sy == '[': 2243 s.next() 2244 is_type = s.sy == ']' 2245 s.put_back(*saved) 2246 2247 dotted_path.reverse() 2248 for p in dotted_path: 2249 s.put_back('IDENT', p) 2250 s.put_back('.', '.') 2251 2252 s.put_back('IDENT', name) 2253 return not is_type and saved[0] 2254 else: 2255 return True 2256 2257 def looking_at_base_type(s): 2258 #print "looking_at_base_type?", s.sy, s.systring, s.position() 2259 return s.sy == 'IDENT' and s.systring in base_type_start_words 2260 2261 def looking_at_dotted_name(s): 2262 if s.sy == 'IDENT': 2263 name = s.systring 2264 s.next() 2265 result = s.sy == '.' 2266 s.put_back('IDENT', name) 2267 return result 2268 else: 2269 return 0 2270 2271 def looking_at_call(s): 2272 "See if we're looking at a.b.c(" 2273 # Don't mess up the original position, so save and restore it. 2274 # Unfortunately there's no good way to handle this, as a subsequent call 2275 # to next() will not advance the position until it reads a new token. 2276 position = s.start_line, s.start_col 2277 result = looking_at_expr(s) == u'(' 2278 if not result: 2279 s.start_line, s.start_col = position 2280 return result 2281 2282 basic_c_type_names = cython.declare( 2283 set, set(["void", "char", "int", "float", "double", "bint"])) 2284 2285 special_basic_c_types = cython.declare(dict, { 2286 # name : (signed, longness) 2287 "Py_UNICODE" : (0, 0), 2288 "Py_UCS4" : (0, 0), 2289 "Py_ssize_t" : (2, 0), 2290 "ssize_t" : (2, 0), 2291 "size_t" : (0, 0), 2292 "ptrdiff_t" : (2, 0), 2293 }) 2294 2295 sign_and_longness_words = cython.declare( 2296 set, set(["short", "long", "signed", "unsigned"])) 2297 2298 base_type_start_words = cython.declare( 2299 set, 2300 basic_c_type_names 2301 | sign_and_longness_words 2302 | set(special_basic_c_types)) 2303 2304 struct_enum_union = cython.declare( 2305 set, set(["struct", "union", "enum", "packed"])) 2306 2307 def p_sign_and_longness(s): 2308 signed = 1 2309 longness = 0 2310 while s.sy == 'IDENT' and s.systring in sign_and_longness_words: 2311 if s.systring == 'unsigned': 2312 signed = 0 2313 elif s.systring == 'signed': 2314 signed = 2 2315 elif s.systring == 'short': 2316 longness = -1 2317 elif s.systring == 'long': 2318 longness += 1 2319 s.next() 2320 return signed, longness 2321 2322 def p_opt_cname(s): 2323 literal = p_opt_string_literal(s, 'u') 2324 if literal is not None: 2325 cname = EncodedString(literal) 2326 cname.encoding = s.source_encoding 2327 else: 2328 cname = None 2329 return cname 2330 2331 def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0, 2332 assignable = 0, nonempty = 0, 2333 calling_convention_allowed = 0): 2334 # If empty is true, the declarator must be empty. If nonempty is true, 2335 # the declarator must be nonempty. Otherwise we don't care. 2336 # If cmethod_flag is true, then if this declarator declares 2337 # a function, it's a C method of an extension type. 2338 pos = s.position() 2339 if s.sy == '(': 2340 s.next() 2341 if s.sy == ')' or looking_at_name(s): 2342 base = Nodes.CNameDeclaratorNode(pos, name = EncodedString(u""), cname = None) 2343 result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag) 2344 else: 2345 result = p_c_declarator(s, ctx, empty = empty, is_type = is_type, 2346 cmethod_flag = cmethod_flag, 2347 nonempty = nonempty, 2348 calling_convention_allowed = 1) 2349 s.expect(')') 2350 else: 2351 result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag, 2352 assignable, nonempty) 2353 if not calling_convention_allowed and result.calling_convention and s.sy != '(': 2354 error(s.position(), "%s on something that is not a function" 2355 % result.calling_convention) 2356 while s.sy in ('[', '('): 2357 pos = s.position() 2358 if s.sy == '[': 2359 result = p_c_array_declarator(s, result) 2360 else: # sy == '(' 2361 s.next() 2362 result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag) 2363 cmethod_flag = 0 2364 return result 2365 2366 def p_c_array_declarator(s, base): 2367 pos = s.position() 2368 s.next() # '[' 2369 if s.sy != ']': 2370 dim = p_testlist(s) 2371 else: 2372 dim = None 2373 s.expect(']') 2374 return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim) 2375 2376 def p_c_func_declarator(s, pos, ctx, base, cmethod_flag): 2377 # Opening paren has already been skipped 2378 args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag, 2379 nonempty_declarators = 0) 2380 ellipsis = p_optional_ellipsis(s) 2381 s.expect(')') 2382 nogil = p_nogil(s) 2383 exc_val, exc_check = p_exception_value_clause(s) 2384 with_gil = p_with_gil(s) 2385 return Nodes.CFuncDeclaratorNode(pos, 2386 base = base, args = args, has_varargs = ellipsis, 2387 exception_value = exc_val, exception_check = exc_check, 2388 nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil) 2389 2390 supported_overloaded_operators = cython.declare(set, set([ 2391 '+', '-', '*', '/', '%', 2392 '++', '--', '~', '|', '&', '^', '<<', '>>', ',', 2393 '==', '!=', '>=', '>', '<=', '<', 2394 '[]', '()', '!', 2395 ])) 2396 2397 def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag, 2398 assignable, nonempty): 2399 pos = s.position() 2400 calling_convention = p_calling_convention(s) 2401 if s.sy == '*': 2402 s.next() 2403 if s.systring == 'const': 2404 const_pos = s.position() 2405 s.next() 2406 const_base = p_c_declarator(s, ctx, empty = empty, 2407 is_type = is_type, 2408 cmethod_flag = cmethod_flag, 2409 assignable = assignable, 2410 nonempty = nonempty) 2411 base = Nodes.CConstDeclaratorNode(const_pos, base = const_base) 2412 else: 2413 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, 2414 cmethod_flag = cmethod_flag, 2415 assignable = assignable, nonempty = nonempty) 2416 result = Nodes.CPtrDeclaratorNode(pos, 2417 base = base) 2418 elif s.sy == '**': # scanner returns this as a single token 2419 s.next() 2420 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, 2421 cmethod_flag = cmethod_flag, 2422 assignable = assignable, nonempty = nonempty) 2423 result = Nodes.CPtrDeclaratorNode(pos, 2424 base = Nodes.CPtrDeclaratorNode(pos, 2425 base = base)) 2426 elif s.sy == '&': 2427 s.next() 2428 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, 2429 cmethod_flag = cmethod_flag, 2430 assignable = assignable, nonempty = nonempty) 2431 result = Nodes.CReferenceDeclaratorNode(pos, base = base) 2432 else: 2433 rhs = None 2434 if s.sy == 'IDENT': 2435 name = EncodedString(s.systring) 2436 if empty: 2437 error(s.position(), "Declarator should be empty") 2438 s.next() 2439 cname = p_opt_cname(s) 2440 if name != 'operator' and s.sy == '=' and assignable: 2441 s.next() 2442 rhs = p_test(s) 2443 else: 2444 if nonempty: 2445 error(s.position(), "Empty declarator") 2446 name = "" 2447 cname = None 2448 if cname is None and ctx.namespace is not None and nonempty: 2449 cname = ctx.namespace + "::" + name 2450 if name == 'operator' and ctx.visibility == 'extern' and nonempty: 2451 op = s.sy 2452 if [1 for c in op if c in '+-*/<=>!%&|([^~,']: 2453 s.next() 2454 # Handle diphthong operators. 2455 if op == '(': 2456 s.expect(')') 2457 op = '()' 2458 elif op == '[': 2459 s.expect(']') 2460 op = '[]' 2461 elif op in ('-', '+', '|', '&') and s.sy == op: 2462 op *= 2 # ++, --, ... 2463 s.next() 2464 elif s.sy == '=': 2465 op += s.sy # +=, -=, ... 2466 s.next() 2467 if op not in supported_overloaded_operators: 2468 s.error("Overloading operator '%s' not yet supported." % op, 2469 fatal=False) 2470 name += op 2471 result = Nodes.CNameDeclaratorNode(pos, 2472 name = name, cname = cname, default = rhs) 2473 result.calling_convention = calling_convention 2474 return result 2475 2476 def p_nogil(s): 2477 if s.sy == 'IDENT' and s.systring == 'nogil': 2478 s.next() 2479 return 1 2480 else: 2481 return 0 2482 2483 def p_with_gil(s): 2484 if s.sy == 'with': 2485 s.next() 2486 s.expect_keyword('gil') 2487 return 1 2488 else: 2489 return 0 2490 2491 def p_exception_value_clause(s): 2492 exc_val = None 2493 exc_check = 0 2494 if s.sy == 'except': 2495 s.next() 2496 if s.sy == '*': 2497 exc_check = 1 2498 s.next() 2499 elif s.sy == '+': 2500 exc_check = '+' 2501 s.next() 2502 if s.sy == 'IDENT': 2503 name = s.systring 2504 s.next() 2505 exc_val = p_name(s, name) 2506 else: 2507 if s.sy == '?': 2508 exc_check = 1 2509 s.next() 2510 exc_val = p_test(s) 2511 return exc_val, exc_check 2512 2513 c_arg_list_terminators = cython.declare(set, set(['*', '**', '.', ')'])) 2514 2515 def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0, 2516 nonempty_declarators = 0, kw_only = 0, annotated = 1): 2517 # Comma-separated list of C argument declarations, possibly empty. 2518 # May have a trailing comma. 2519 args = [] 2520 is_self_arg = cmethod_flag 2521 while s.sy not in c_arg_list_terminators: 2522 args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg, 2523 nonempty = nonempty_declarators, kw_only = kw_only, 2524 annotated = annotated)) 2525 if s.sy != ',': 2526 break 2527 s.next() 2528 is_self_arg = 0 2529 return args 2530 2531 def p_optional_ellipsis(s): 2532 if s.sy == '.': 2533 expect_ellipsis(s) 2534 return 1 2535 else: 2536 return 0 2537 2538 def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, 2539 kw_only = 0, annotated = 1): 2540 pos = s.position() 2541 not_none = or_none = 0 2542 default = None 2543 annotation = None 2544 if s.in_python_file: 2545 # empty type declaration 2546 base_type = Nodes.CSimpleBaseTypeNode(pos, 2547 name = None, module_path = [], 2548 is_basic_c_type = 0, signed = 0, 2549 complex = 0, longness = 0, 2550 is_self_arg = cmethod_flag, templates = None) 2551 else: 2552 base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty) 2553 declarator = p_c_declarator(s, ctx, nonempty = nonempty) 2554 if s.sy in ('not', 'or') and not s.in_python_file: 2555 kind = s.sy 2556 s.next() 2557 if s.sy == 'IDENT' and s.systring == 'None': 2558 s.next() 2559 else: 2560 s.error("Expected 'None'") 2561 if not in_pyfunc: 2562 error(pos, "'%s None' only allowed in Python functions" % kind) 2563 or_none = kind == 'or' 2564 not_none = kind == 'not' 2565 if annotated and s.sy == ':': 2566 s.next() 2567 annotation = p_test(s) 2568 if s.sy == '=': 2569 s.next() 2570 if 'pxd' in ctx.level: 2571 if s.sy not in ['*', '?']: 2572 error(pos, "default values cannot be specified in pxd files, use ? or *") 2573 default = ExprNodes.BoolNode(1) 2574 s.next() 2575 else: 2576 default = p_test(s) 2577 return Nodes.CArgDeclNode(pos, 2578 base_type = base_type, 2579 declarator = declarator, 2580 not_none = not_none, 2581 or_none = or_none, 2582 default = default, 2583 annotation = annotation, 2584 kw_only = kw_only) 2585 2586 def p_api(s): 2587 if s.sy == 'IDENT' and s.systring == 'api': 2588 s.next() 2589 return 1 2590 else: 2591 return 0 2592 2593 def p_cdef_statement(s, ctx): 2594 pos = s.position() 2595 ctx.visibility = p_visibility(s, ctx.visibility) 2596 ctx.api = ctx.api or p_api(s) 2597 if ctx.api: 2598 if ctx.visibility not in ('private', 'public'): 2599 error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility) 2600 if (ctx.visibility == 'extern') and s.sy == 'from': 2601 return p_cdef_extern_block(s, pos, ctx) 2602 elif s.sy == 'import': 2603 s.next() 2604 return p_cdef_extern_block(s, pos, ctx) 2605 elif p_nogil(s): 2606 ctx.nogil = 1 2607 if ctx.overridable: 2608 error(pos, "cdef blocks cannot be declared cpdef") 2609 return p_cdef_block(s, ctx) 2610 elif s.sy == ':': 2611 if ctx.overridable: 2612 error(pos, "cdef blocks cannot be declared cpdef") 2613 return p_cdef_block(s, ctx) 2614 elif s.sy == 'class': 2615 if ctx.level not in ('module', 'module_pxd'): 2616 error(pos, "Extension type definition not allowed here") 2617 if ctx.overridable: 2618 error(pos, "Extension types cannot be declared cpdef") 2619 return p_c_class_definition(s, pos, ctx) 2620 elif s.sy == 'IDENT' and s.systring == 'cppclass': 2621 return p_cpp_class_definition(s, pos, ctx) 2622 elif s.sy == 'IDENT' and s.systring in struct_enum_union: 2623 if ctx.level not in ('module', 'module_pxd'): 2624 error(pos, "C struct/union/enum definition not allowed here") 2625 if ctx.overridable: 2626 error(pos, "C struct/union/enum cannot be declared cpdef") 2627 return p_struct_enum(s, pos, ctx) 2628 elif s.sy == 'IDENT' and s.systring == 'fused': 2629 return p_fused_definition(s, pos, ctx) 2630 else: 2631 return p_c_func_or_var_declaration(s, pos, ctx) 2632 2633 def p_cdef_block(s, ctx): 2634 return p_suite(s, ctx(cdef_flag = 1)) 2635 2636 def p_cdef_extern_block(s, pos, ctx): 2637 if ctx.overridable: 2638 error(pos, "cdef extern blocks cannot be declared cpdef") 2639 include_file = None 2640 s.expect('from') 2641 if s.sy == '*': 2642 s.next() 2643 else: 2644 include_file = p_string_literal(s, 'u')[2] 2645 ctx = ctx(cdef_flag = 1, visibility = 'extern') 2646 if s.systring == "namespace": 2647 s.next() 2648 ctx.namespace = p_string_literal(s, 'u')[2] 2649 if p_nogil(s): 2650 ctx.nogil = 1 2651 body = p_suite(s, ctx) 2652 return Nodes.CDefExternNode(pos, 2653 include_file = include_file, 2654 body = body, 2655 namespace = ctx.namespace) 2656 2657 def p_c_enum_definition(s, pos, ctx): 2658 # s.sy == ident 'enum' 2659 s.next() 2660 if s.sy == 'IDENT': 2661 name = s.systring 2662 s.next() 2663 cname = p_opt_cname(s) 2664 if cname is None and ctx.namespace is not None: 2665 cname = ctx.namespace + "::" + name 2666 else: 2667 name = None 2668 cname = None 2669 items = None 2670 s.expect(':') 2671 items = [] 2672 if s.sy != 'NEWLINE': 2673 p_c_enum_line(s, ctx, items) 2674 else: 2675 s.next() # 'NEWLINE' 2676 s.expect_indent() 2677 while s.sy not in ('DEDENT', 'EOF'): 2678 p_c_enum_line(s, ctx, items) 2679 s.expect_dedent() 2680 return Nodes.CEnumDefNode( 2681 pos, name = name, cname = cname, items = items, 2682 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility, 2683 api = ctx.api, in_pxd = ctx.level == 'module_pxd') 2684 2685 def p_c_enum_line(s, ctx, items): 2686 if s.sy != 'pass': 2687 p_c_enum_item(s, ctx, items) 2688 while s.sy == ',': 2689 s.next() 2690 if s.sy in ('NEWLINE', 'EOF'): 2691 break 2692 p_c_enum_item(s, ctx, items) 2693 else: 2694 s.next() 2695 s.expect_newline("Syntax error in enum item list") 2696 2697 def p_c_enum_item(s, ctx, items): 2698 pos = s.position() 2699 name = p_ident(s) 2700 cname = p_opt_cname(s) 2701 if cname is None and ctx.namespace is not None: 2702 cname = ctx.namespace + "::" + name 2703 value = None 2704 if s.sy == '=': 2705 s.next() 2706 value = p_test(s) 2707 items.append(Nodes.CEnumDefItemNode(pos, 2708 name = name, cname = cname, value = value)) 2709 2710 def p_c_struct_or_union_definition(s, pos, ctx): 2711 packed = False 2712 if s.systring == 'packed': 2713 packed = True 2714 s.next() 2715 if s.sy != 'IDENT' or s.systring != 'struct': 2716 s.expected('struct') 2717 # s.sy == ident 'struct' or 'union' 2718 kind = s.systring 2719 s.next() 2720 name = p_ident(s) 2721 cname = p_opt_cname(s) 2722 if cname is None and ctx.namespace is not None: 2723 cname = ctx.namespace + "::" + name 2724 attributes = None 2725 if s.sy == ':': 2726 s.next() 2727 s.expect('NEWLINE') 2728 s.expect_indent() 2729 attributes = [] 2730 body_ctx = Ctx() 2731 while s.sy != 'DEDENT': 2732 if s.sy != 'pass': 2733 attributes.append( 2734 p_c_func_or_var_declaration(s, s.position(), body_ctx)) 2735 else: 2736 s.next() 2737 s.expect_newline("Expected a newline") 2738 s.expect_dedent() 2739 else: 2740 s.expect_newline("Syntax error in struct or union definition") 2741 return Nodes.CStructOrUnionDefNode(pos, 2742 name = name, cname = cname, kind = kind, attributes = attributes, 2743 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility, 2744 api = ctx.api, in_pxd = ctx.level == 'module_pxd', packed = packed) 2745 2746 def p_fused_definition(s, pos, ctx): 2747 """ 2748 c(type)def fused my_fused_type: 2749 ... 2750 """ 2751 # s.systring == 'fused' 2752 2753 if ctx.level not in ('module', 'module_pxd'): 2754 error(pos, "Fused type definition not allowed here") 2755 2756 s.next() 2757 name = p_ident(s) 2758 2759 s.expect(":") 2760 s.expect_newline() 2761 s.expect_indent() 2762 2763 types = [] 2764 while s.sy != 'DEDENT': 2765 if s.sy != 'pass': 2766 #types.append(p_c_declarator(s)) 2767 types.append(p_c_base_type(s)) #, nonempty=1)) 2768 else: 2769 s.next() 2770 2771 s.expect_newline() 2772 2773 s.expect_dedent() 2774 2775 if not types: 2776 error(pos, "Need at least one type") 2777 2778 return Nodes.FusedTypeNode(pos, name=name, types=types) 2779 2780 def p_struct_enum(s, pos, ctx): 2781 if s.systring == 'enum': 2782 return p_c_enum_definition(s, pos, ctx) 2783 else: 2784 return p_c_struct_or_union_definition(s, pos, ctx) 2785 2786 def p_visibility(s, prev_visibility): 2787 pos = s.position() 2788 visibility = prev_visibility 2789 if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'): 2790 visibility = s.systring 2791 if prev_visibility != 'private' and visibility != prev_visibility: 2792 s.error("Conflicting visibility options '%s' and '%s'" 2793 % (prev_visibility, visibility), fatal=False) 2794 s.next() 2795 return visibility 2796 2797 def p_c_modifiers(s): 2798 if s.sy == 'IDENT' and s.systring in ('inline',): 2799 modifier = s.systring 2800 s.next() 2801 return [modifier] + p_c_modifiers(s) 2802 return [] 2803 2804 def p_c_func_or_var_declaration(s, pos, ctx): 2805 cmethod_flag = ctx.level in ('c_class', 'c_class_pxd') 2806 modifiers = p_c_modifiers(s) 2807 base_type = p_c_base_type(s, nonempty = 1, templates = ctx.templates) 2808 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag, 2809 assignable = 1, nonempty = 1) 2810 declarator.overridable = ctx.overridable 2811 if s.sy == 'IDENT' and s.systring == 'const' and ctx.level == 'cpp_class': 2812 s.next() 2813 is_const_method = 1 2814 else: 2815 is_const_method = 0 2816 if s.sy == ':': 2817 if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd', 'cpp_class') and not ctx.templates: 2818 s.error("C function definition not allowed here") 2819 doc, suite = p_suite_with_docstring(s, Ctx(level='function')) 2820 result = Nodes.CFuncDefNode(pos, 2821 visibility = ctx.visibility, 2822 base_type = base_type, 2823 declarator = declarator, 2824 body = suite, 2825 doc = doc, 2826 modifiers = modifiers, 2827 api = ctx.api, 2828 overridable = ctx.overridable, 2829 is_const_method = is_const_method) 2830 else: 2831 #if api: 2832 # s.error("'api' not allowed with variable declaration") 2833 if is_const_method: 2834 declarator.is_const_method = is_const_method 2835 declarators = [declarator] 2836 while s.sy == ',': 2837 s.next() 2838 if s.sy == 'NEWLINE': 2839 break 2840 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag, 2841 assignable = 1, nonempty = 1) 2842 declarators.append(declarator) 2843 doc_line = s.start_line + 1 2844 s.expect_newline("Syntax error in C variable declaration") 2845 if ctx.level in ('c_class', 'c_class_pxd') and s.start_line == doc_line: 2846 doc = p_doc_string(s) 2847 else: 2848 doc = None 2849 result = Nodes.CVarDefNode(pos, 2850 visibility = ctx.visibility, 2851 base_type = base_type, 2852 declarators = declarators, 2853 in_pxd = ctx.level in ('module_pxd', 'c_class_pxd'), 2854 doc = doc, 2855 api = ctx.api, 2856 modifiers = modifiers, 2857 overridable = ctx.overridable) 2858 return result 2859 2860 def p_ctypedef_statement(s, ctx): 2861 # s.sy == 'ctypedef' 2862 pos = s.position() 2863 s.next() 2864 visibility = p_visibility(s, ctx.visibility) 2865 api = p_api(s) 2866 ctx = ctx(typedef_flag = 1, visibility = visibility) 2867 if api: 2868 ctx.api = 1 2869 if s.sy == 'class': 2870 return p_c_class_definition(s, pos, ctx) 2871 elif s.sy == 'IDENT' and s.systring in struct_enum_union: 2872 return p_struct_enum(s, pos, ctx) 2873 elif s.sy == 'IDENT' and s.systring == 'fused': 2874 return p_fused_definition(s, pos, ctx) 2875 else: 2876 base_type = p_c_base_type(s, nonempty = 1) 2877 declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1) 2878 s.expect_newline("Syntax error in ctypedef statement") 2879 return Nodes.CTypeDefNode( 2880 pos, base_type = base_type, 2881 declarator = declarator, 2882 visibility = visibility, api = api, 2883 in_pxd = ctx.level == 'module_pxd') 2884 2885 def p_decorators(s): 2886 decorators = [] 2887 while s.sy == 'DECORATOR': 2888 pos = s.position() 2889 s.next() 2890 decstring = p_dotted_name(s, as_allowed=0)[2] 2891 names = decstring.split('.') 2892 decorator = ExprNodes.NameNode(pos, name=EncodedString(names[0])) 2893 for name in names[1:]: 2894 decorator = ExprNodes.AttributeNode(pos, 2895 attribute=EncodedString(name), 2896 obj=decorator) 2897 if s.sy == '(': 2898 decorator = p_call(s, decorator) 2899 decorators.append(Nodes.DecoratorNode(pos, decorator=decorator)) 2900 s.expect_newline("Expected a newline after decorator") 2901 return decorators 2902 2903 def p_def_statement(s, decorators=None): 2904 # s.sy == 'def' 2905 pos = s.position() 2906 s.next() 2907 name = EncodedString( p_ident(s) ) 2908 s.expect('(') 2909 args, star_arg, starstar_arg = p_varargslist(s, terminator=')') 2910 s.expect(')') 2911 if p_nogil(s): 2912 error(pos, "Python function cannot be declared nogil") 2913 return_type_annotation = None 2914 if s.sy == '->': 2915 s.next() 2916 return_type_annotation = p_test(s) 2917 doc, body = p_suite_with_docstring(s, Ctx(level='function')) 2918 return Nodes.DefNode(pos, name = name, args = args, 2919 star_arg = star_arg, starstar_arg = starstar_arg, 2920 doc = doc, body = body, decorators = decorators, 2921 return_type_annotation = return_type_annotation) 2922 2923 def p_varargslist(s, terminator=')', annotated=1): 2924 args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1, 2925 annotated = annotated) 2926 star_arg = None 2927 starstar_arg = None 2928 if s.sy == '*': 2929 s.next() 2930 if s.sy == 'IDENT': 2931 star_arg = p_py_arg_decl(s, annotated=annotated) 2932 if s.sy == ',': 2933 s.next() 2934 args.extend(p_c_arg_list(s, in_pyfunc = 1, 2935 nonempty_declarators = 1, kw_only = 1, annotated = annotated)) 2936 elif s.sy != terminator: 2937 s.error("Syntax error in Python function argument list") 2938 if s.sy == '**': 2939 s.next() 2940 starstar_arg = p_py_arg_decl(s, annotated=annotated) 2941 return (args, star_arg, starstar_arg) 2942 2943 def p_py_arg_decl(s, annotated = 1): 2944 pos = s.position() 2945 name = p_ident(s) 2946 annotation = None 2947 if annotated and s.sy == ':': 2948 s.next() 2949 annotation = p_test(s) 2950 return Nodes.PyArgDeclNode(pos, name = name, annotation = annotation) 2951 2952 def p_class_statement(s, decorators): 2953 # s.sy == 'class' 2954 pos = s.position() 2955 s.next() 2956 class_name = EncodedString( p_ident(s) ) 2957 class_name.encoding = s.source_encoding 2958 arg_tuple = None 2959 keyword_dict = None 2960 starstar_arg = None 2961 if s.sy == '(': 2962 positional_args, keyword_args, star_arg, starstar_arg = \ 2963 p_call_parse_args(s, allow_genexp = False) 2964 arg_tuple, keyword_dict = p_call_build_packed_args( 2965 pos, positional_args, keyword_args, star_arg, None) 2966 if arg_tuple is None: 2967 # XXX: empty arg_tuple 2968 arg_tuple = ExprNodes.TupleNode(pos, args=[]) 2969 doc, body = p_suite_with_docstring(s, Ctx(level='class')) 2970 return Nodes.PyClassDefNode( 2971 pos, name=class_name, 2972 bases=arg_tuple, 2973 keyword_args=keyword_dict, 2974 starstar_arg=starstar_arg, 2975 doc=doc, body=body, decorators=decorators, 2976 force_py3_semantics=s.context.language_level >= 3) 2977 2978 def p_c_class_definition(s, pos, ctx): 2979 # s.sy == 'class' 2980 s.next() 2981 module_path = [] 2982 class_name = p_ident(s) 2983 while s.sy == '.': 2984 s.next() 2985 module_path.append(class_name) 2986 class_name = p_ident(s) 2987 if module_path and ctx.visibility != 'extern': 2988 error(pos, "Qualified class name only allowed for 'extern' C class") 2989 if module_path and s.sy == 'IDENT' and s.systring == 'as': 2990 s.next() 2991 as_name = p_ident(s) 2992 else: 2993 as_name = class_name 2994 objstruct_name = None 2995 typeobj_name = None 2996 base_class_module = None 2997 base_class_name = None 2998 if s.sy == '(': 2999 s.next() 3000 base_class_path = [p_ident(s)] 3001 while s.sy == '.': 3002 s.next() 3003 base_class_path.append(p_ident(s)) 3004 if s.sy == ',': 3005 s.error("C class may only have one base class", fatal=False) 3006 s.expect(')') 3007 base_class_module = ".".join(base_class_path[:-1]) 3008 base_class_name = base_class_path[-1] 3009 if s.sy == '[': 3010 if ctx.visibility not in ('public', 'extern') and not ctx.api: 3011 error(s.position(), "Name options only allowed for 'public', 'api', or 'extern' C class") 3012 objstruct_name, typeobj_name = p_c_class_options(s) 3013 if s.sy == ':': 3014 if ctx.level == 'module_pxd': 3015 body_level = 'c_class_pxd' 3016 else: 3017 body_level = 'c_class' 3018 doc, body = p_suite_with_docstring(s, Ctx(level=body_level)) 3019 else: 3020 s.expect_newline("Syntax error in C class definition") 3021 doc = None 3022 body = None 3023 if ctx.visibility == 'extern': 3024 if not module_path: 3025 error(pos, "Module name required for 'extern' C class") 3026 if typeobj_name: 3027 error(pos, "Type object name specification not allowed for 'extern' C class") 3028 elif ctx.visibility == 'public': 3029 if not objstruct_name: 3030 error(pos, "Object struct name specification required for 'public' C class") 3031 if not typeobj_name: 3032 error(pos, "Type object name specification required for 'public' C class") 3033 elif ctx.visibility == 'private': 3034 if ctx.api: 3035 if not objstruct_name: 3036 error(pos, "Object struct name specification required for 'api' C class") 3037 if not typeobj_name: 3038 error(pos, "Type object name specification required for 'api' C class") 3039 else: 3040 error(pos, "Invalid class visibility '%s'" % ctx.visibility) 3041 return Nodes.CClassDefNode(pos, 3042 visibility = ctx.visibility, 3043 typedef_flag = ctx.typedef_flag, 3044 api = ctx.api, 3045 module_name = ".".join(module_path), 3046 class_name = class_name, 3047 as_name = as_name, 3048 base_class_module = base_class_module, 3049 base_class_name = base_class_name, 3050 objstruct_name = objstruct_name, 3051 typeobj_name = typeobj_name, 3052 in_pxd = ctx.level == 'module_pxd', 3053 doc = doc, 3054 body = body) 3055 3056 def p_c_class_options(s): 3057 objstruct_name = None 3058 typeobj_name = None 3059 s.expect('[') 3060 while 1: 3061 if s.sy != 'IDENT': 3062 break 3063 if s.systring == 'object': 3064 s.next() 3065 objstruct_name = p_ident(s) 3066 elif s.systring == 'type': 3067 s.next() 3068 typeobj_name = p_ident(s) 3069 if s.sy != ',': 3070 break 3071 s.next() 3072 s.expect(']', "Expected 'object' or 'type'") 3073 return objstruct_name, typeobj_name 3074 3075 3076 def p_property_decl(s): 3077 pos = s.position() 3078 s.next() # 'property' 3079 name = p_ident(s) 3080 doc, body = p_suite_with_docstring( 3081 s, Ctx(level='property'), with_doc_only=True) 3082 return Nodes.PropertyNode(pos, name=name, doc=doc, body=body) 3083 3084 3085 def p_ignorable_statement(s): 3086 """ 3087 Parses any kind of ignorable statement that is allowed in .pxd files. 3088 """ 3089 if s.sy == 'BEGIN_STRING': 3090 pos = s.position() 3091 string_node = p_atom(s) 3092 if s.sy != 'EOF': 3093 s.expect_newline("Syntax error in string") 3094 return Nodes.ExprStatNode(pos, expr=string_node) 3095 return None 3096 3097 3098 def p_doc_string(s): 3099 if s.sy == 'BEGIN_STRING': 3100 pos = s.position() 3101 kind, bytes_result, unicode_result = p_cat_string_literal(s) 3102 if s.sy != 'EOF': 3103 s.expect_newline("Syntax error in doc string") 3104 if kind in ('u', ''): 3105 return unicode_result 3106 warning(pos, "Python 3 requires docstrings to be unicode strings") 3107 return bytes_result 3108 else: 3109 return None 3110 3111 3112 def _extract_docstring(node): 3113 """ 3114 Extract a docstring from a statement or from the first statement 3115 in a list. Remove the statement if found. Return a tuple 3116 (plain-docstring or None, node). 3117 """ 3118 doc_node = None 3119 if node is None: 3120 pass 3121 elif isinstance(node, Nodes.ExprStatNode): 3122 if node.expr.is_string_literal: 3123 doc_node = node.expr 3124 node = Nodes.StatListNode(node.pos, stats=[]) 3125 elif isinstance(node, Nodes.StatListNode) and node.stats: 3126 stats = node.stats 3127 if isinstance(stats[0], Nodes.ExprStatNode): 3128 if stats[0].expr.is_string_literal: 3129 doc_node = stats[0].expr 3130 del stats[0] 3131 3132 if doc_node is None: 3133 doc = None 3134 elif isinstance(doc_node, ExprNodes.BytesNode): 3135 warning(node.pos, 3136 "Python 3 requires docstrings to be unicode strings") 3137 doc = doc_node.value 3138 elif isinstance(doc_node, ExprNodes.StringNode): 3139 doc = doc_node.unicode_value 3140 if doc is None: 3141 doc = doc_node.value 3142 else: 3143 doc = doc_node.value 3144 return doc, node 3145 3146 3147 def p_code(s, level=None, ctx=Ctx): 3148 body = p_statement_list(s, ctx(level = level), first_statement = 1) 3149 if s.sy != 'EOF': 3150 s.error("Syntax error in statement [%s,%s]" % ( 3151 repr(s.sy), repr(s.systring))) 3152 return body 3153 3154 _match_compiler_directive_comment = cython.declare(object, re.compile( 3155 r"^#\s*cython\s*:\s*((\w|[.])+\s*=.*)$").match) 3156 3157 def p_compiler_directive_comments(s): 3158 result = {} 3159 while s.sy == 'commentline': 3160 m = _match_compiler_directive_comment(s.systring) 3161 if m: 3162 directives = m.group(1).strip() 3163 try: 3164 result.update(Options.parse_directive_list( 3165 directives, ignore_unknown=True)) 3166 except ValueError, e: 3167 s.error(e.args[0], fatal=False) 3168 s.next() 3169 return result 3170 3171 def p_module(s, pxd, full_module_name, ctx=Ctx): 3172 pos = s.position() 3173 3174 directive_comments = p_compiler_directive_comments(s) 3175 s.parse_comments = False 3176 3177 if 'language_level' in directive_comments: 3178 s.context.set_language_level(directive_comments['language_level']) 3179 3180 doc = p_doc_string(s) 3181 if pxd: 3182 level = 'module_pxd' 3183 else: 3184 level = 'module' 3185 3186 body = p_statement_list(s, ctx(level=level), first_statement = 1) 3187 if s.sy != 'EOF': 3188 s.error("Syntax error in statement [%s,%s]" % ( 3189 repr(s.sy), repr(s.systring))) 3190 return ModuleNode(pos, doc = doc, body = body, 3191 full_module_name = full_module_name, 3192 directive_comments = directive_comments) 3193 3194 def p_cpp_class_definition(s, pos, ctx): 3195 # s.sy == 'cppclass' 3196 s.next() 3197 module_path = [] 3198 class_name = p_ident(s) 3199 cname = p_opt_cname(s) 3200 if cname is None and ctx.namespace is not None: 3201 cname = ctx.namespace + "::" + class_name 3202 if s.sy == '.': 3203 error(pos, "Qualified class name not allowed C++ class") 3204 if s.sy == '[': 3205 s.next() 3206 templates = [p_ident(s)] 3207 while s.sy == ',': 3208 s.next() 3209 templates.append(p_ident(s)) 3210 s.expect(']') 3211 else: 3212 templates = None 3213 if s.sy == '(': 3214 s.next() 3215 base_classes = [p_c_base_type(s, templates = templates)] 3216 while s.sy == ',': 3217 s.next() 3218 base_classes.append(p_c_base_type(s, templates = templates)) 3219 s.expect(')') 3220 else: 3221 base_classes = [] 3222 if s.sy == '[': 3223 error(s.position(), "Name options not allowed for C++ class") 3224 nogil = p_nogil(s) 3225 if s.sy == ':': 3226 s.next() 3227 s.expect('NEWLINE') 3228 s.expect_indent() 3229 attributes = [] 3230 body_ctx = Ctx(visibility = ctx.visibility, level='cpp_class', nogil=nogil or ctx.nogil) 3231 body_ctx.templates = templates 3232 while s.sy != 'DEDENT': 3233 if s.systring == 'cppclass': 3234 attributes.append( 3235 p_cpp_class_definition(s, s.position(), body_ctx)) 3236 elif s.sy != 'pass': 3237 attributes.append( 3238 p_c_func_or_var_declaration(s, s.position(), body_ctx)) 3239 else: 3240 s.next() 3241 s.expect_newline("Expected a newline") 3242 s.expect_dedent() 3243 else: 3244 attributes = None 3245 s.expect_newline("Syntax error in C++ class definition") 3246 return Nodes.CppClassNode(pos, 3247 name = class_name, 3248 cname = cname, 3249 base_classes = base_classes, 3250 visibility = ctx.visibility, 3251 in_pxd = ctx.level == 'module_pxd', 3252 attributes = attributes, 3253 templates = templates) 3254 3255 3256 3257 #---------------------------------------------- 3258 # 3259 # Debugging 3260 # 3261 #---------------------------------------------- 3262 3263 def print_parse_tree(f, node, level, key = None): 3264 from types import ListType, TupleType 3265 from Nodes import Node 3266 ind = " " * level 3267 if node: 3268 f.write(ind) 3269 if key: 3270 f.write("%s: " % key) 3271 t = type(node) 3272 if t is tuple: 3273 f.write("(%s @ %s\n" % (node[0], node[1])) 3274 for i in xrange(2, len(node)): 3275 print_parse_tree(f, node[i], level+1) 3276 f.write("%s)\n" % ind) 3277 return 3278 elif isinstance(node, Node): 3279 try: 3280 tag = node.tag 3281 except AttributeError: 3282 tag = node.__class__.__name__ 3283 f.write("%s @ %s\n" % (tag, node.pos)) 3284 for name, value in node.__dict__.items(): 3285 if name != 'tag' and name != 'pos': 3286 print_parse_tree(f, value, level+1, name) 3287 return 3288 elif t is list: 3289 f.write("[\n") 3290 for i in xrange(len(node)): 3291 print_parse_tree(f, node[i], level+1) 3292 f.write("%s]\n" % ind) 3293 return 3294 f.write("%s%s\n" % (ind, node)) 3295