1 /* parsermodule.c 2 * 3 * Copyright 1995-1996 by Fred L. Drake, Jr. and Virginia Polytechnic 4 * Institute and State University, Blacksburg, Virginia, USA. 5 * Portions copyright 1991-1995 by Stichting Mathematisch Centrum, 6 * Amsterdam, The Netherlands. Copying is permitted under the terms 7 * associated with the main Python distribution, with the additional 8 * restriction that this additional notice be included and maintained 9 * on all distributed copies. 10 * 11 * This module serves to replace the original parser module written 12 * by Guido. The functionality is not matched precisely, but the 13 * original may be implemented on top of this. This is desirable 14 * since the source of the text to be parsed is now divorced from 15 * this interface. 16 * 17 * Unlike the prior interface, the ability to give a parse tree 18 * produced by Python code as a tuple to the compiler is enabled by 19 * this module. See the documentation for more details. 20 * 21 * I've added some annotations that help with the lint code-checking 22 * program, but they're not complete by a long shot. The real errors 23 * that lint detects are gone, but there are still warnings with 24 * Py_[X]DECREF() and Py_[X]INCREF() macros. The lint annotations 25 * look like "NOTE(...)". 26 */ 27 28 #include "Python.h" /* general Python API */ 29 #include "Python-ast.h" /* mod_ty */ 30 #include "graminit.h" /* symbols defined in the grammar */ 31 #include "node.h" /* internal parser structure */ 32 #include "errcode.h" /* error codes for PyNode_*() */ 33 #include "token.h" /* token definitions */ 34 #include "grammar.h" 35 #include "parsetok.h" 36 /* ISTERMINAL() / ISNONTERMINAL() */ 37 #include "compile.h" 38 #undef Yield 39 #include "ast.h" 40 #include "pyarena.h" 41 42 extern grammar _PyParser_Grammar; /* From graminit.c */ 43 44 #ifdef lint 45 #include <note.h> 46 #else 47 #define NOTE(x) 48 #endif 49 50 /* String constants used to initialize module attributes. 51 * 52 */ 53 static char parser_copyright_string[] = 54 "Copyright 1995-1996 by Virginia Polytechnic Institute & State\n\ 55 University, Blacksburg, Virginia, USA, and Fred L. Drake, Jr., Reston,\n\ 56 Virginia, USA. Portions copyright 1991-1995 by Stichting Mathematisch\n\ 57 Centrum, Amsterdam, The Netherlands."; 58 59 60 PyDoc_STRVAR(parser_doc_string, 61 "This is an interface to Python's internal parser."); 62 63 static char parser_version_string[] = "0.5"; 64 65 66 typedef PyObject* (*SeqMaker) (Py_ssize_t length); 67 typedef int (*SeqInserter) (PyObject* sequence, 68 Py_ssize_t index, 69 PyObject* element); 70 71 /* The function below is copyrighted by Stichting Mathematisch Centrum. The 72 * original copyright statement is included below, and continues to apply 73 * in full to the function immediately following. All other material is 74 * original, copyrighted by Fred L. Drake, Jr. and Virginia Polytechnic 75 * Institute and State University. Changes were made to comply with the 76 * new naming conventions. Added arguments to provide support for creating 77 * lists as well as tuples, and optionally including the line numbers. 78 */ 79 80 81 static PyObject* 82 node2tuple(node *n, /* node to convert */ 83 SeqMaker mkseq, /* create sequence */ 84 SeqInserter addelem, /* func. to add elem. in seq. */ 85 int lineno, /* include line numbers? */ 86 int col_offset) /* include column offsets? */ 87 { 88 if (n == NULL) { 89 Py_INCREF(Py_None); 90 return (Py_None); 91 } 92 if (ISNONTERMINAL(TYPE(n))) { 93 int i; 94 PyObject *v; 95 PyObject *w; 96 97 v = mkseq(1 + NCH(n) + (TYPE(n) == encoding_decl)); 98 if (v == NULL) 99 return (v); 100 w = PyInt_FromLong(TYPE(n)); 101 if (w == NULL) { 102 Py_DECREF(v); 103 return ((PyObject*) NULL); 104 } 105 (void) addelem(v, 0, w); 106 for (i = 0; i < NCH(n); i++) { 107 w = node2tuple(CHILD(n, i), mkseq, addelem, lineno, col_offset); 108 if (w == NULL) { 109 Py_DECREF(v); 110 return ((PyObject*) NULL); 111 } 112 (void) addelem(v, i+1, w); 113 } 114 115 if (TYPE(n) == encoding_decl) 116 (void) addelem(v, i+1, PyString_FromString(STR(n))); 117 return (v); 118 } 119 else if (ISTERMINAL(TYPE(n))) { 120 PyObject *result = mkseq(2 + lineno + col_offset); 121 if (result != NULL) { 122 (void) addelem(result, 0, PyInt_FromLong(TYPE(n))); 123 (void) addelem(result, 1, PyString_FromString(STR(n))); 124 if (lineno == 1) 125 (void) addelem(result, 2, PyInt_FromLong(n->n_lineno)); 126 if (col_offset == 1) 127 (void) addelem(result, 3, PyInt_FromLong(n->n_col_offset)); 128 } 129 return (result); 130 } 131 else { 132 PyErr_SetString(PyExc_SystemError, 133 "unrecognized parse tree node type"); 134 return ((PyObject*) NULL); 135 } 136 } 137 /* 138 * End of material copyrighted by Stichting Mathematisch Centrum. 139 */ 140 141 142 143 /* There are two types of intermediate objects we're interested in: 144 * 'eval' and 'exec' types. These constants can be used in the st_type 145 * field of the object type to identify which any given object represents. 146 * These should probably go in an external header to allow other extensions 147 * to use them, but then, we really should be using C++ too. ;-) 148 */ 149 150 #define PyST_EXPR 1 151 #define PyST_SUITE 2 152 153 154 /* These are the internal objects and definitions required to implement the 155 * ST type. Most of the internal names are more reminiscent of the 'old' 156 * naming style, but the code uses the new naming convention. 157 */ 158 159 static PyObject* 160 parser_error = 0; 161 162 163 typedef struct { 164 PyObject_HEAD /* standard object header */ 165 node* st_node; /* the node* returned by the parser */ 166 int st_type; /* EXPR or SUITE ? */ 167 PyCompilerFlags st_flags; /* Parser and compiler flags */ 168 } PyST_Object; 169 170 171 static void parser_free(PyST_Object *st); 172 static PyObject* parser_sizeof(PyST_Object *, void *); 173 static int parser_compare(PyST_Object *left, PyST_Object *right); 174 static PyObject *parser_getattr(PyObject *self, char *name); 175 static PyObject* parser_compilest(PyST_Object *, PyObject *, PyObject *); 176 static PyObject* parser_isexpr(PyST_Object *, PyObject *, PyObject *); 177 static PyObject* parser_issuite(PyST_Object *, PyObject *, PyObject *); 178 static PyObject* parser_st2list(PyST_Object *, PyObject *, PyObject *); 179 static PyObject* parser_st2tuple(PyST_Object *, PyObject *, PyObject *); 180 181 #define PUBLIC_METHOD_TYPE (METH_VARARGS|METH_KEYWORDS) 182 183 static PyMethodDef 184 parser_methods[] = { 185 {"compile", (PyCFunction)parser_compilest, PUBLIC_METHOD_TYPE, 186 PyDoc_STR("Compile this ST object into a code object.")}, 187 {"isexpr", (PyCFunction)parser_isexpr, PUBLIC_METHOD_TYPE, 188 PyDoc_STR("Determines if this ST object was created from an expression.")}, 189 {"issuite", (PyCFunction)parser_issuite, PUBLIC_METHOD_TYPE, 190 PyDoc_STR("Determines if this ST object was created from a suite.")}, 191 {"tolist", (PyCFunction)parser_st2list, PUBLIC_METHOD_TYPE, 192 PyDoc_STR("Creates a list-tree representation of this ST.")}, 193 {"totuple", (PyCFunction)parser_st2tuple, PUBLIC_METHOD_TYPE, 194 PyDoc_STR("Creates a tuple-tree representation of this ST.")}, 195 {"__sizeof__", (PyCFunction)parser_sizeof, METH_NOARGS, 196 PyDoc_STR("Returns size in memory, in bytes.")}, 197 {NULL, NULL, 0, NULL} 198 }; 199 200 static 201 PyTypeObject PyST_Type = { 202 PyVarObject_HEAD_INIT(NULL, 0) 203 "parser.st", /* tp_name */ 204 (int) sizeof(PyST_Object), /* tp_basicsize */ 205 0, /* tp_itemsize */ 206 (destructor)parser_free, /* tp_dealloc */ 207 0, /* tp_print */ 208 parser_getattr, /* tp_getattr */ 209 0, /* tp_setattr */ 210 (cmpfunc)parser_compare, /* tp_compare */ 211 0, /* tp_repr */ 212 0, /* tp_as_number */ 213 0, /* tp_as_sequence */ 214 0, /* tp_as_mapping */ 215 0, /* tp_hash */ 216 0, /* tp_call */ 217 0, /* tp_str */ 218 0, /* tp_getattro */ 219 0, /* tp_setattro */ 220 221 /* Functions to access object as input/output buffer */ 222 0, /* tp_as_buffer */ 223 224 Py_TPFLAGS_DEFAULT, /* tp_flags */ 225 226 /* __doc__ */ 227 "Intermediate representation of a Python parse tree.", 228 0, /* tp_traverse */ 229 0, /* tp_clear */ 230 0, /* tp_richcompare */ 231 0, /* tp_weaklistoffset */ 232 0, /* tp_iter */ 233 0, /* tp_iternext */ 234 parser_methods, /* tp_methods */ 235 }; /* PyST_Type */ 236 237 238 static int 239 parser_compare_nodes(node *left, node *right) 240 { 241 int j; 242 243 if (TYPE(left) < TYPE(right)) 244 return (-1); 245 246 if (TYPE(right) < TYPE(left)) 247 return (1); 248 249 if (ISTERMINAL(TYPE(left))) 250 return (strcmp(STR(left), STR(right))); 251 252 if (NCH(left) < NCH(right)) 253 return (-1); 254 255 if (NCH(right) < NCH(left)) 256 return (1); 257 258 for (j = 0; j < NCH(left); ++j) { 259 int v = parser_compare_nodes(CHILD(left, j), CHILD(right, j)); 260 261 if (v != 0) 262 return (v); 263 } 264 return (0); 265 } 266 267 268 /* int parser_compare(PyST_Object* left, PyST_Object* right) 269 * 270 * Comparison function used by the Python operators ==, !=, <, >, <=, >= 271 * This really just wraps a call to parser_compare_nodes() with some easy 272 * checks and protection code. 273 * 274 */ 275 static int 276 parser_compare(PyST_Object *left, PyST_Object *right) 277 { 278 if (left == right) 279 return (0); 280 281 if ((left == 0) || (right == 0)) 282 return (-1); 283 284 return (parser_compare_nodes(left->st_node, right->st_node)); 285 } 286 287 288 /* parser_newstobject(node* st) 289 * 290 * Allocates a new Python object representing an ST. This is simply the 291 * 'wrapper' object that holds a node* and allows it to be passed around in 292 * Python code. 293 * 294 */ 295 static PyObject* 296 parser_newstobject(node *st, int type) 297 { 298 PyST_Object* o = PyObject_New(PyST_Object, &PyST_Type); 299 300 if (o != 0) { 301 o->st_node = st; 302 o->st_type = type; 303 o->st_flags.cf_flags = 0; 304 } 305 else { 306 PyNode_Free(st); 307 } 308 return ((PyObject*)o); 309 } 310 311 312 /* void parser_free(PyST_Object* st) 313 * 314 * This is called by a del statement that reduces the reference count to 0. 315 * 316 */ 317 static void 318 parser_free(PyST_Object *st) 319 { 320 PyNode_Free(st->st_node); 321 PyObject_Del(st); 322 } 323 324 325 /* parser_st2tuple(PyObject* self, PyObject* args, PyObject* kw) 326 * 327 * This provides conversion from a node* to a tuple object that can be 328 * returned to the Python-level caller. The ST object is not modified. 329 * 330 */ 331 static PyObject* 332 parser_st2tuple(PyST_Object *self, PyObject *args, PyObject *kw) 333 { 334 PyObject *line_option = 0; 335 PyObject *col_option = 0; 336 PyObject *res = 0; 337 int ok; 338 339 static char *keywords[] = {"ast", "line_info", "col_info", NULL}; 340 341 if (self == NULL) { 342 ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|OO:st2tuple", keywords, 343 &PyST_Type, &self, &line_option, 344 &col_option); 345 } 346 else 347 ok = PyArg_ParseTupleAndKeywords(args, kw, "|OO:totuple", &keywords[1], 348 &line_option, &col_option); 349 if (ok != 0) { 350 int lineno = 0; 351 int col_offset = 0; 352 if (line_option != NULL) { 353 lineno = PyObject_IsTrue(line_option); 354 if (lineno < 0) 355 return NULL; 356 } 357 if (col_option != NULL) { 358 col_offset = PyObject_IsTrue(col_option); 359 if (col_offset < 0) 360 return NULL; 361 } 362 /* 363 * Convert ST into a tuple representation. Use Guido's function, 364 * since it's known to work already. 365 */ 366 res = node2tuple(((PyST_Object*)self)->st_node, 367 PyTuple_New, PyTuple_SetItem, lineno, col_offset); 368 } 369 return (res); 370 } 371 372 static PyObject* 373 parser_ast2tuple(PyST_Object *self, PyObject *args, PyObject *kw) 374 { 375 if (PyErr_WarnPy3k("ast2tuple is removed in 3.x; use st2tuple", 1) < 0) 376 return NULL; 377 return parser_st2tuple(self, args, kw); 378 } 379 380 381 /* parser_st2list(PyObject* self, PyObject* args, PyObject* kw) 382 * 383 * This provides conversion from a node* to a list object that can be 384 * returned to the Python-level caller. The ST object is not modified. 385 * 386 */ 387 static PyObject* 388 parser_st2list(PyST_Object *self, PyObject *args, PyObject *kw) 389 { 390 PyObject *line_option = 0; 391 PyObject *col_option = 0; 392 PyObject *res = 0; 393 int ok; 394 395 static char *keywords[] = {"ast", "line_info", "col_info", NULL}; 396 397 if (self == NULL) 398 ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|OO:st2list", keywords, 399 &PyST_Type, &self, &line_option, 400 &col_option); 401 else 402 ok = PyArg_ParseTupleAndKeywords(args, kw, "|OO:tolist", &keywords[1], 403 &line_option, &col_option); 404 if (ok) { 405 int lineno = 0; 406 int col_offset = 0; 407 if (line_option != 0) { 408 lineno = PyObject_IsTrue(line_option); 409 if (lineno < 0) 410 return NULL; 411 } 412 if (col_option != 0) { 413 col_offset = PyObject_IsTrue(col_option); 414 if (col_offset < 0) 415 return NULL; 416 } 417 /* 418 * Convert ST into a tuple representation. Use Guido's function, 419 * since it's known to work already. 420 */ 421 res = node2tuple(self->st_node, 422 PyList_New, PyList_SetItem, lineno, col_offset); 423 } 424 return (res); 425 } 426 427 static PyObject* 428 parser_ast2list(PyST_Object *self, PyObject *args, PyObject *kw) 429 { 430 if (PyErr_WarnPy3k("ast2list is removed in 3.x; use st2list", 1) < 0) 431 return NULL; 432 return parser_st2list(self, args, kw); 433 } 434 435 436 /* parser_compilest(PyObject* self, PyObject* args) 437 * 438 * This function creates code objects from the parse tree represented by 439 * the passed-in data object. An optional file name is passed in as well. 440 * 441 */ 442 static PyObject* 443 parser_compilest(PyST_Object *self, PyObject *args, PyObject *kw) 444 { 445 PyObject* res = 0; 446 PyArena* arena; 447 mod_ty mod; 448 char* str = "<syntax-tree>"; 449 int ok; 450 451 static char *keywords[] = {"ast", "filename", NULL}; 452 453 if (self == NULL) 454 ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|s:compilest", keywords, 455 &PyST_Type, &self, &str); 456 else 457 ok = PyArg_ParseTupleAndKeywords(args, kw, "|s:compile", &keywords[1], 458 &str); 459 460 if (ok) { 461 arena = PyArena_New(); 462 if (arena) { 463 mod = PyAST_FromNode(self->st_node, &(self->st_flags), str, arena); 464 if (mod) { 465 res = (PyObject *)PyAST_Compile(mod, str, &(self->st_flags), arena); 466 } 467 PyArena_Free(arena); 468 } 469 } 470 471 return (res); 472 } 473 474 static PyObject* 475 parser_compileast(PyST_Object *self, PyObject *args, PyObject *kw) 476 { 477 if (PyErr_WarnPy3k("compileast is removed in 3.x; use compilest", 1) < 0) 478 return NULL; 479 return parser_compilest(self, args, kw); 480 } 481 482 483 /* PyObject* parser_isexpr(PyObject* self, PyObject* args) 484 * PyObject* parser_issuite(PyObject* self, PyObject* args) 485 * 486 * Checks the passed-in ST object to determine if it is an expression or 487 * a statement suite, respectively. The return is a Python truth value. 488 * 489 */ 490 static PyObject* 491 parser_isexpr(PyST_Object *self, PyObject *args, PyObject *kw) 492 { 493 PyObject* res = 0; 494 int ok; 495 496 static char *keywords[] = {"ast", NULL}; 497 498 if (self == NULL) 499 ok = PyArg_ParseTupleAndKeywords(args, kw, "O!:isexpr", keywords, 500 &PyST_Type, &self); 501 else 502 ok = PyArg_ParseTupleAndKeywords(args, kw, ":isexpr", &keywords[1]); 503 504 if (ok) { 505 /* Check to see if the ST represents an expression or not. */ 506 res = (self->st_type == PyST_EXPR) ? Py_True : Py_False; 507 Py_INCREF(res); 508 } 509 return (res); 510 } 511 512 513 static PyObject* 514 parser_issuite(PyST_Object *self, PyObject *args, PyObject *kw) 515 { 516 PyObject* res = 0; 517 int ok; 518 519 static char *keywords[] = {"ast", NULL}; 520 521 if (self == NULL) 522 ok = PyArg_ParseTupleAndKeywords(args, kw, "O!:issuite", keywords, 523 &PyST_Type, &self); 524 else 525 ok = PyArg_ParseTupleAndKeywords(args, kw, ":issuite", &keywords[1]); 526 527 if (ok) { 528 /* Check to see if the ST represents an expression or not. */ 529 res = (self->st_type == PyST_EXPR) ? Py_False : Py_True; 530 Py_INCREF(res); 531 } 532 return (res); 533 } 534 535 536 static PyObject* 537 parser_getattr(PyObject *self, char *name) 538 { 539 return (Py_FindMethod(parser_methods, self, name)); 540 } 541 542 543 /* err_string(char* message) 544 * 545 * Sets the error string for an exception of type ParserError. 546 * 547 */ 548 static void 549 err_string(char *message) 550 { 551 PyErr_SetString(parser_error, message); 552 } 553 554 555 /* PyObject* parser_do_parse(PyObject* args, int type) 556 * 557 * Internal function to actually execute the parse and return the result if 558 * successful or set an exception if not. 559 * 560 */ 561 static PyObject* 562 parser_do_parse(PyObject *args, PyObject *kw, char *argspec, int type) 563 { 564 char* string = 0; 565 PyObject* res = 0; 566 int flags = 0; 567 perrdetail err; 568 569 static char *keywords[] = {"source", NULL}; 570 571 if (PyArg_ParseTupleAndKeywords(args, kw, argspec, keywords, &string)) { 572 node* n = PyParser_ParseStringFlagsFilenameEx(string, NULL, 573 &_PyParser_Grammar, 574 (type == PyST_EXPR) 575 ? eval_input : file_input, 576 &err, &flags); 577 578 if (n) { 579 res = parser_newstobject(n, type); 580 if (res) 581 ((PyST_Object *)res)->st_flags.cf_flags = flags & PyCF_MASK; 582 } 583 else 584 PyParser_SetError(&err); 585 } 586 return (res); 587 } 588 589 590 /* PyObject* parser_expr(PyObject* self, PyObject* args) 591 * PyObject* parser_suite(PyObject* self, PyObject* args) 592 * 593 * External interfaces to the parser itself. Which is called determines if 594 * the parser attempts to recognize an expression ('eval' form) or statement 595 * suite ('exec' form). The real work is done by parser_do_parse() above. 596 * 597 */ 598 static PyObject* 599 parser_expr(PyST_Object *self, PyObject *args, PyObject *kw) 600 { 601 NOTE(ARGUNUSED(self)) 602 return (parser_do_parse(args, kw, "s:expr", PyST_EXPR)); 603 } 604 605 606 static PyObject* 607 parser_suite(PyST_Object *self, PyObject *args, PyObject *kw) 608 { 609 NOTE(ARGUNUSED(self)) 610 return (parser_do_parse(args, kw, "s:suite", PyST_SUITE)); 611 } 612 613 614 615 /* This is the messy part of the code. Conversion from a tuple to an ST 616 * object requires that the input tuple be valid without having to rely on 617 * catching an exception from the compiler. This is done to allow the 618 * compiler itself to remain fast, since most of its input will come from 619 * the parser directly, and therefore be known to be syntactically correct. 620 * This validation is done to ensure that we don't core dump the compile 621 * phase, returning an exception instead. 622 * 623 * Two aspects can be broken out in this code: creating a node tree from 624 * the tuple passed in, and verifying that it is indeed valid. It may be 625 * advantageous to expand the number of ST types to include funcdefs and 626 * lambdadefs to take advantage of the optimizer, recognizing those STs 627 * here. They are not necessary, and not quite as useful in a raw form. 628 * For now, let's get expressions and suites working reliably. 629 */ 630 631 632 static node* build_node_tree(PyObject *tuple); 633 static int validate_expr_tree(node *tree); 634 static int validate_file_input(node *tree); 635 static int validate_encoding_decl(node *tree); 636 637 /* PyObject* parser_tuple2st(PyObject* self, PyObject* args) 638 * 639 * This is the public function, called from the Python code. It receives a 640 * single tuple object from the caller, and creates an ST object if the 641 * tuple can be validated. It does this by checking the first code of the 642 * tuple, and, if acceptable, builds the internal representation. If this 643 * step succeeds, the internal representation is validated as fully as 644 * possible with the various validate_*() routines defined below. 645 * 646 * This function must be changed if support is to be added for PyST_FRAGMENT 647 * ST objects. 648 * 649 */ 650 static PyObject* 651 parser_tuple2st(PyST_Object *self, PyObject *args, PyObject *kw) 652 { 653 NOTE(ARGUNUSED(self)) 654 PyObject *st = 0; 655 PyObject *tuple; 656 node *tree; 657 658 static char *keywords[] = {"sequence", NULL}; 659 660 if (!PyArg_ParseTupleAndKeywords(args, kw, "O:sequence2st", keywords, 661 &tuple)) 662 return (0); 663 if (!PySequence_Check(tuple)) { 664 PyErr_SetString(PyExc_ValueError, 665 "sequence2st() requires a single sequence argument"); 666 return (0); 667 } 668 /* 669 * Convert the tree to the internal form before checking it. 670 */ 671 tree = build_node_tree(tuple); 672 if (tree != 0) { 673 int start_sym = TYPE(tree); 674 if (start_sym == eval_input) { 675 /* Might be an eval form. */ 676 if (validate_expr_tree(tree)) 677 st = parser_newstobject(tree, PyST_EXPR); 678 else 679 PyNode_Free(tree); 680 } 681 else if (start_sym == file_input) { 682 /* This looks like an exec form so far. */ 683 if (validate_file_input(tree)) 684 st = parser_newstobject(tree, PyST_SUITE); 685 else 686 PyNode_Free(tree); 687 } 688 else if (start_sym == encoding_decl) { 689 /* This looks like an encoding_decl so far. */ 690 if (validate_encoding_decl(tree)) 691 st = parser_newstobject(tree, PyST_SUITE); 692 else 693 PyNode_Free(tree); 694 } 695 else { 696 /* This is a fragment, at best. */ 697 PyNode_Free(tree); 698 err_string("parse tree does not use a valid start symbol"); 699 } 700 } 701 /* Make sure we raise an exception on all errors. We should never 702 * get this, but we'd do well to be sure something is done. 703 */ 704 if (st == NULL && !PyErr_Occurred()) 705 err_string("unspecified ST error occurred"); 706 707 return st; 708 } 709 710 static PyObject* 711 parser_tuple2ast(PyST_Object *self, PyObject *args, PyObject *kw) 712 { 713 if (PyErr_WarnPy3k("tuple2ast is removed in 3.x; use tuple2st", 1) < 0) 714 return NULL; 715 return parser_tuple2st(self, args, kw); 716 } 717 718 static PyObject * 719 parser_sizeof(PyST_Object *st, void *unused) 720 { 721 Py_ssize_t res; 722 723 res = _PyObject_SIZE(Py_TYPE(st)) + _PyNode_SizeOf(st->st_node); 724 return PyLong_FromSsize_t(res); 725 } 726 727 728 /* node* build_node_children() 729 * 730 * Iterate across the children of the current non-terminal node and build 731 * their structures. If successful, return the root of this portion of 732 * the tree, otherwise, 0. Any required exception will be specified already, 733 * and no memory will have been deallocated. 734 * 735 */ 736 static node* 737 build_node_children(PyObject *tuple, node *root, int *line_num) 738 { 739 Py_ssize_t len = PyObject_Size(tuple); 740 Py_ssize_t i; 741 int err; 742 743 for (i = 1; i < len; ++i) { 744 /* elem must always be a sequence, however simple */ 745 PyObject* elem = PySequence_GetItem(tuple, i); 746 int ok = elem != NULL; 747 long type = 0; 748 char *strn = 0; 749 750 if (ok) 751 ok = PySequence_Check(elem); 752 if (ok) { 753 PyObject *temp = PySequence_GetItem(elem, 0); 754 if (temp == NULL) 755 ok = 0; 756 else { 757 ok = PyInt_Check(temp); 758 if (ok) 759 type = PyInt_AS_LONG(temp); 760 Py_DECREF(temp); 761 } 762 } 763 if (!ok) { 764 PyObject *err = Py_BuildValue("os", elem, 765 "Illegal node construct."); 766 PyErr_SetObject(parser_error, err); 767 Py_XDECREF(err); 768 Py_XDECREF(elem); 769 return (0); 770 } 771 if (ISTERMINAL(type)) { 772 Py_ssize_t len = PyObject_Size(elem); 773 PyObject *temp; 774 775 if ((len != 2) && (len != 3)) { 776 err_string("terminal nodes must have 2 or 3 entries"); 777 return 0; 778 } 779 temp = PySequence_GetItem(elem, 1); 780 if (temp == NULL) 781 return 0; 782 if (!PyString_Check(temp)) { 783 PyErr_Format(parser_error, 784 "second item in terminal node must be a string," 785 " found %s", 786 Py_TYPE(temp)->tp_name); 787 Py_DECREF(temp); 788 return 0; 789 } 790 if (len == 3) { 791 PyObject *o = PySequence_GetItem(elem, 2); 792 if (o != NULL) { 793 if (PyInt_Check(o)) 794 *line_num = PyInt_AS_LONG(o); 795 else { 796 PyErr_Format(parser_error, 797 "third item in terminal node must be an" 798 " integer, found %s", 799 Py_TYPE(temp)->tp_name); 800 Py_DECREF(o); 801 Py_DECREF(temp); 802 return 0; 803 } 804 Py_DECREF(o); 805 } 806 } 807 len = PyString_GET_SIZE(temp) + 1; 808 strn = (char *)PyObject_MALLOC(len); 809 if (strn != NULL) 810 (void) memcpy(strn, PyString_AS_STRING(temp), len); 811 Py_DECREF(temp); 812 } 813 else if (!ISNONTERMINAL(type)) { 814 /* 815 * It has to be one or the other; this is an error. 816 * Raise an exception. 817 */ 818 PyObject *err = Py_BuildValue("os", elem, "unknown node type."); 819 PyErr_SetObject(parser_error, err); 820 Py_XDECREF(err); 821 Py_XDECREF(elem); 822 return (0); 823 } 824 err = PyNode_AddChild(root, type, strn, *line_num, 0); 825 if (err == E_NOMEM) { 826 PyObject_FREE(strn); 827 return (node *) PyErr_NoMemory(); 828 } 829 if (err == E_OVERFLOW) { 830 PyObject_FREE(strn); 831 PyErr_SetString(PyExc_ValueError, 832 "unsupported number of child nodes"); 833 return NULL; 834 } 835 836 if (ISNONTERMINAL(type)) { 837 node* new_child = CHILD(root, i - 1); 838 839 if (new_child != build_node_children(elem, new_child, line_num)) { 840 Py_XDECREF(elem); 841 return (0); 842 } 843 } 844 else if (type == NEWLINE) { /* It's true: we increment the */ 845 ++(*line_num); /* line number *after* the newline! */ 846 } 847 Py_XDECREF(elem); 848 } 849 return root; 850 } 851 852 853 static node* 854 build_node_tree(PyObject *tuple) 855 { 856 node* res = 0; 857 PyObject *temp = PySequence_GetItem(tuple, 0); 858 long num = -1; 859 860 if (temp != NULL) 861 num = PyInt_AsLong(temp); 862 Py_XDECREF(temp); 863 if (ISTERMINAL(num)) { 864 /* 865 * The tuple is simple, but it doesn't start with a start symbol. 866 * Raise an exception now and be done with it. 867 */ 868 tuple = Py_BuildValue("os", tuple, 869 "Illegal syntax-tree; cannot start with terminal symbol."); 870 PyErr_SetObject(parser_error, tuple); 871 Py_XDECREF(tuple); 872 } 873 else if (ISNONTERMINAL(num)) { 874 /* 875 * Not efficient, but that can be handled later. 876 */ 877 int line_num = 0; 878 PyObject *encoding = NULL; 879 880 if (num == encoding_decl) { 881 encoding = PySequence_GetItem(tuple, 2); 882 /* tuple isn't borrowed anymore here, need to DECREF */ 883 tuple = PySequence_GetSlice(tuple, 0, 2); 884 } 885 res = PyNode_New(num); 886 if (res != NULL) { 887 if (res != build_node_children(tuple, res, &line_num)) { 888 PyNode_Free(res); 889 res = NULL; 890 } 891 if (res && encoding) { 892 Py_ssize_t len; 893 len = PyString_GET_SIZE(encoding) + 1; 894 res->n_str = (char *)PyObject_MALLOC(len); 895 if (res->n_str != NULL) 896 (void) memcpy(res->n_str, PyString_AS_STRING(encoding), len); 897 Py_DECREF(encoding); 898 Py_DECREF(tuple); 899 } 900 } 901 } 902 else { 903 /* The tuple is illegal -- if the number is neither TERMINAL nor 904 * NONTERMINAL, we can't use it. Not sure the implementation 905 * allows this condition, but the API doesn't preclude it. 906 */ 907 PyObject *err = Py_BuildValue("os", tuple, 908 "Illegal component tuple."); 909 PyErr_SetObject(parser_error, err); 910 Py_XDECREF(err); 911 } 912 913 return (res); 914 } 915 916 917 /* 918 * Validation routines used within the validation section: 919 */ 920 static int validate_terminal(node *terminal, int type, char *string); 921 922 #define validate_ampersand(ch) validate_terminal(ch, AMPER, "&") 923 #define validate_circumflex(ch) validate_terminal(ch, CIRCUMFLEX, "^") 924 #define validate_colon(ch) validate_terminal(ch, COLON, ":") 925 #define validate_comma(ch) validate_terminal(ch, COMMA, ",") 926 #define validate_dedent(ch) validate_terminal(ch, DEDENT, "") 927 #define validate_equal(ch) validate_terminal(ch, EQUAL, "=") 928 #define validate_indent(ch) validate_terminal(ch, INDENT, (char*)NULL) 929 #define validate_lparen(ch) validate_terminal(ch, LPAR, "(") 930 #define validate_newline(ch) validate_terminal(ch, NEWLINE, (char*)NULL) 931 #define validate_rparen(ch) validate_terminal(ch, RPAR, ")") 932 #define validate_semi(ch) validate_terminal(ch, SEMI, ";") 933 #define validate_star(ch) validate_terminal(ch, STAR, "*") 934 #define validate_vbar(ch) validate_terminal(ch, VBAR, "|") 935 #define validate_doublestar(ch) validate_terminal(ch, DOUBLESTAR, "**") 936 #define validate_dot(ch) validate_terminal(ch, DOT, ".") 937 #define validate_at(ch) validate_terminal(ch, AT, "@") 938 #define validate_name(ch, str) validate_terminal(ch, NAME, str) 939 940 #define VALIDATER(n) static int validate_##n(node *tree) 941 942 VALIDATER(node); VALIDATER(small_stmt); 943 VALIDATER(class); VALIDATER(node); 944 VALIDATER(parameters); VALIDATER(suite); 945 VALIDATER(testlist); VALIDATER(varargslist); 946 VALIDATER(fpdef); VALIDATER(fplist); 947 VALIDATER(stmt); VALIDATER(simple_stmt); 948 VALIDATER(expr_stmt); VALIDATER(power); 949 VALIDATER(print_stmt); VALIDATER(del_stmt); 950 VALIDATER(return_stmt); VALIDATER(list_iter); 951 VALIDATER(raise_stmt); VALIDATER(import_stmt); 952 VALIDATER(import_name); VALIDATER(import_from); 953 VALIDATER(global_stmt); VALIDATER(list_if); 954 VALIDATER(assert_stmt); VALIDATER(list_for); 955 VALIDATER(exec_stmt); VALIDATER(compound_stmt); 956 VALIDATER(while); VALIDATER(for); 957 VALIDATER(try); VALIDATER(except_clause); 958 VALIDATER(test); VALIDATER(and_test); 959 VALIDATER(not_test); VALIDATER(comparison); 960 VALIDATER(comp_op); VALIDATER(expr); 961 VALIDATER(xor_expr); VALIDATER(and_expr); 962 VALIDATER(shift_expr); VALIDATER(arith_expr); 963 VALIDATER(term); VALIDATER(factor); 964 VALIDATER(atom); VALIDATER(lambdef); 965 VALIDATER(trailer); VALIDATER(subscript); 966 VALIDATER(subscriptlist); VALIDATER(sliceop); 967 VALIDATER(exprlist); VALIDATER(dictorsetmaker); 968 VALIDATER(arglist); VALIDATER(argument); 969 VALIDATER(listmaker); VALIDATER(yield_stmt); 970 VALIDATER(testlist1); VALIDATER(comp_for); 971 VALIDATER(comp_iter); VALIDATER(comp_if); 972 VALIDATER(testlist_comp); VALIDATER(yield_expr); 973 VALIDATER(yield_or_testlist); VALIDATER(or_test); 974 VALIDATER(old_test); VALIDATER(old_lambdef); 975 976 #undef VALIDATER 977 978 #define is_even(n) (((n) & 1) == 0) 979 #define is_odd(n) (((n) & 1) == 1) 980 981 982 static int 983 validate_ntype(node *n, int t) 984 { 985 if (TYPE(n) != t) { 986 PyErr_Format(parser_error, "Expected node type %d, got %d.", 987 t, TYPE(n)); 988 return 0; 989 } 990 return 1; 991 } 992 993 994 /* Verifies that the number of child nodes is exactly 'num', raising 995 * an exception if it isn't. The exception message does not indicate 996 * the exact number of nodes, allowing this to be used to raise the 997 * "right" exception when the wrong number of nodes is present in a 998 * specific variant of a statement's syntax. This is commonly used 999 * in that fashion. 1000 */ 1001 static int 1002 validate_numnodes(node *n, int num, const char *const name) 1003 { 1004 if (NCH(n) != num) { 1005 PyErr_Format(parser_error, 1006 "Illegal number of children for %s node.", name); 1007 return 0; 1008 } 1009 return 1; 1010 } 1011 1012 1013 static int 1014 validate_terminal(node *terminal, int type, char *string) 1015 { 1016 int res = (validate_ntype(terminal, type) 1017 && ((string == 0) || (strcmp(string, STR(terminal)) == 0))); 1018 1019 if (!res && !PyErr_Occurred()) { 1020 PyErr_Format(parser_error, 1021 "Illegal terminal: expected \"%s\"", string); 1022 } 1023 return (res); 1024 } 1025 1026 1027 /* X (',' X) [','] 1028 */ 1029 static int 1030 validate_repeating_list(node *tree, int ntype, int (*vfunc)(node *), 1031 const char *const name) 1032 { 1033 int nch = NCH(tree); 1034 int res = (nch && validate_ntype(tree, ntype) 1035 && vfunc(CHILD(tree, 0))); 1036 1037 if (!res && !PyErr_Occurred()) 1038 (void) validate_numnodes(tree, 1, name); 1039 else { 1040 if (is_even(nch)) 1041 res = validate_comma(CHILD(tree, --nch)); 1042 if (res && nch > 1) { 1043 int pos = 1; 1044 for ( ; res && pos < nch; pos += 2) 1045 res = (validate_comma(CHILD(tree, pos)) 1046 && vfunc(CHILD(tree, pos + 1))); 1047 } 1048 } 1049 return (res); 1050 } 1051 1052 1053 /* validate_class() 1054 * 1055 * classdef: 1056 * 'class' NAME ['(' testlist ')'] ':' suite 1057 */ 1058 static int 1059 validate_class(node *tree) 1060 { 1061 int nch = NCH(tree); 1062 int res = (validate_ntype(tree, classdef) && 1063 ((nch == 4) || (nch == 6) || (nch == 7))); 1064 1065 if (res) { 1066 res = (validate_name(CHILD(tree, 0), "class") 1067 && validate_ntype(CHILD(tree, 1), NAME) 1068 && validate_colon(CHILD(tree, nch - 2)) 1069 && validate_suite(CHILD(tree, nch - 1))); 1070 } 1071 else { 1072 (void) validate_numnodes(tree, 4, "class"); 1073 } 1074 1075 if (res) { 1076 if (nch == 7) { 1077 res = ((validate_lparen(CHILD(tree, 2)) && 1078 validate_testlist(CHILD(tree, 3)) && 1079 validate_rparen(CHILD(tree, 4)))); 1080 } 1081 else if (nch == 6) { 1082 res = (validate_lparen(CHILD(tree,2)) && 1083 validate_rparen(CHILD(tree,3))); 1084 } 1085 } 1086 return (res); 1087 } 1088 1089 1090 /* if_stmt: 1091 * 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] 1092 */ 1093 static int 1094 validate_if(node *tree) 1095 { 1096 int nch = NCH(tree); 1097 int res = (validate_ntype(tree, if_stmt) 1098 && (nch >= 4) 1099 && validate_name(CHILD(tree, 0), "if") 1100 && validate_test(CHILD(tree, 1)) 1101 && validate_colon(CHILD(tree, 2)) 1102 && validate_suite(CHILD(tree, 3))); 1103 1104 if (res && ((nch % 4) == 3)) { 1105 /* ... 'else' ':' suite */ 1106 res = (validate_name(CHILD(tree, nch - 3), "else") 1107 && validate_colon(CHILD(tree, nch - 2)) 1108 && validate_suite(CHILD(tree, nch - 1))); 1109 nch -= 3; 1110 } 1111 else if (!res && !PyErr_Occurred()) 1112 (void) validate_numnodes(tree, 4, "if"); 1113 if ((nch % 4) != 0) 1114 /* Will catch the case for nch < 4 */ 1115 res = validate_numnodes(tree, 0, "if"); 1116 else if (res && (nch > 4)) { 1117 /* ... ('elif' test ':' suite)+ ... */ 1118 int j = 4; 1119 while ((j < nch) && res) { 1120 res = (validate_name(CHILD(tree, j), "elif") 1121 && validate_colon(CHILD(tree, j + 2)) 1122 && validate_test(CHILD(tree, j + 1)) 1123 && validate_suite(CHILD(tree, j + 3))); 1124 j += 4; 1125 } 1126 } 1127 return (res); 1128 } 1129 1130 1131 /* parameters: 1132 * '(' [varargslist] ')' 1133 * 1134 */ 1135 static int 1136 validate_parameters(node *tree) 1137 { 1138 int nch = NCH(tree); 1139 int res = validate_ntype(tree, parameters) && ((nch == 2) || (nch == 3)); 1140 1141 if (res) { 1142 res = (validate_lparen(CHILD(tree, 0)) 1143 && validate_rparen(CHILD(tree, nch - 1))); 1144 if (res && (nch == 3)) 1145 res = validate_varargslist(CHILD(tree, 1)); 1146 } 1147 else { 1148 (void) validate_numnodes(tree, 2, "parameters"); 1149 } 1150 return (res); 1151 } 1152 1153 1154 /* validate_suite() 1155 * 1156 * suite: 1157 * simple_stmt 1158 * | NEWLINE INDENT stmt+ DEDENT 1159 */ 1160 static int 1161 validate_suite(node *tree) 1162 { 1163 int nch = NCH(tree); 1164 int res = (validate_ntype(tree, suite) && ((nch == 1) || (nch >= 4))); 1165 1166 if (res && (nch == 1)) 1167 res = validate_simple_stmt(CHILD(tree, 0)); 1168 else if (res) { 1169 /* NEWLINE INDENT stmt+ DEDENT */ 1170 res = (validate_newline(CHILD(tree, 0)) 1171 && validate_indent(CHILD(tree, 1)) 1172 && validate_stmt(CHILD(tree, 2)) 1173 && validate_dedent(CHILD(tree, nch - 1))); 1174 1175 if (res && (nch > 4)) { 1176 int i = 3; 1177 --nch; /* forget the DEDENT */ 1178 for ( ; res && (i < nch); ++i) 1179 res = validate_stmt(CHILD(tree, i)); 1180 } 1181 else if (nch < 4) 1182 res = validate_numnodes(tree, 4, "suite"); 1183 } 1184 return (res); 1185 } 1186 1187 1188 static int 1189 validate_testlist(node *tree) 1190 { 1191 return (validate_repeating_list(tree, testlist, 1192 validate_test, "testlist")); 1193 } 1194 1195 1196 static int 1197 validate_testlist1(node *tree) 1198 { 1199 return (validate_repeating_list(tree, testlist1, 1200 validate_test, "testlist1")); 1201 } 1202 1203 1204 static int 1205 validate_testlist_safe(node *tree) 1206 { 1207 return (validate_repeating_list(tree, testlist_safe, 1208 validate_old_test, "testlist_safe")); 1209 } 1210 1211 1212 /* '*' NAME [',' '**' NAME] | '**' NAME 1213 */ 1214 static int 1215 validate_varargslist_trailer(node *tree, int start) 1216 { 1217 int nch = NCH(tree); 1218 int res = 0; 1219 int sym; 1220 1221 if (nch <= start) { 1222 err_string("expected variable argument trailer for varargslist"); 1223 return 0; 1224 } 1225 sym = TYPE(CHILD(tree, start)); 1226 if (sym == STAR) { 1227 /* 1228 * ('*' NAME [',' '**' NAME] 1229 */ 1230 if (nch-start == 2) 1231 res = validate_name(CHILD(tree, start+1), NULL); 1232 else if (nch-start == 5) 1233 res = (validate_name(CHILD(tree, start+1), NULL) 1234 && validate_comma(CHILD(tree, start+2)) 1235 && validate_doublestar(CHILD(tree, start+3)) 1236 && validate_name(CHILD(tree, start+4), NULL)); 1237 } 1238 else if (sym == DOUBLESTAR) { 1239 /* 1240 * '**' NAME 1241 */ 1242 if (nch-start == 2) 1243 res = validate_name(CHILD(tree, start+1), NULL); 1244 } 1245 if (!res) 1246 err_string("illegal variable argument trailer for varargslist"); 1247 return res; 1248 } 1249 1250 1251 /* validate_varargslist() 1252 * 1253 * varargslist: 1254 * (fpdef ['=' test] ',')* 1255 * ('*' NAME [',' '**' NAME] 1256 * | '**' NAME) 1257 * | fpdef ['=' test] (',' fpdef ['=' test])* [','] 1258 * 1259 */ 1260 static int 1261 validate_varargslist(node *tree) 1262 { 1263 int nch = NCH(tree); 1264 int res = validate_ntype(tree, varargslist) && (nch != 0); 1265 int sym; 1266 1267 if (!res) 1268 return 0; 1269 if (nch < 1) { 1270 err_string("varargslist missing child nodes"); 1271 return 0; 1272 } 1273 sym = TYPE(CHILD(tree, 0)); 1274 if (sym == STAR || sym == DOUBLESTAR) 1275 /* whole thing matches: 1276 * '*' NAME [',' '**' NAME] | '**' NAME 1277 */ 1278 res = validate_varargslist_trailer(tree, 0); 1279 else if (sym == fpdef) { 1280 int i = 0; 1281 1282 sym = TYPE(CHILD(tree, nch-1)); 1283 if (sym == NAME) { 1284 /* 1285 * (fpdef ['=' test] ',')+ 1286 * ('*' NAME [',' '**' NAME] 1287 * | '**' NAME) 1288 */ 1289 /* skip over (fpdef ['=' test] ',')+ */ 1290 while (res && (i+2 <= nch)) { 1291 res = validate_fpdef(CHILD(tree, i)); 1292 ++i; 1293 if (res && TYPE(CHILD(tree, i)) == EQUAL && (i+2 <= nch)) { 1294 res = (validate_equal(CHILD(tree, i)) 1295 && validate_test(CHILD(tree, i+1))); 1296 if (res) 1297 i += 2; 1298 } 1299 if (res && i < nch) { 1300 res = validate_comma(CHILD(tree, i)); 1301 ++i; 1302 if (res && i < nch 1303 && (TYPE(CHILD(tree, i)) == DOUBLESTAR 1304 || TYPE(CHILD(tree, i)) == STAR)) 1305 break; 1306 } 1307 } 1308 /* ... '*' NAME [',' '**' NAME] | '**' NAME 1309 * i --^^^ 1310 */ 1311 if (res) 1312 res = validate_varargslist_trailer(tree, i); 1313 } 1314 else { 1315 /* 1316 * fpdef ['=' test] (',' fpdef ['=' test])* [','] 1317 */ 1318 /* strip trailing comma node */ 1319 if (sym == COMMA) { 1320 res = validate_comma(CHILD(tree, nch-1)); 1321 if (!res) 1322 return 0; 1323 --nch; 1324 } 1325 /* 1326 * fpdef ['=' test] (',' fpdef ['=' test])* 1327 */ 1328 res = validate_fpdef(CHILD(tree, 0)); 1329 ++i; 1330 if (res && (i+2 <= nch) && TYPE(CHILD(tree, i)) == EQUAL) { 1331 res = (validate_equal(CHILD(tree, i)) 1332 && validate_test(CHILD(tree, i+1))); 1333 i += 2; 1334 } 1335 /* 1336 * ... (',' fpdef ['=' test])* 1337 * i ---^^^ 1338 */ 1339 while (res && (nch - i) >= 2) { 1340 res = (validate_comma(CHILD(tree, i)) 1341 && validate_fpdef(CHILD(tree, i+1))); 1342 i += 2; 1343 if (res && (nch - i) >= 2 && TYPE(CHILD(tree, i)) == EQUAL) { 1344 res = (validate_equal(CHILD(tree, i)) 1345 && validate_test(CHILD(tree, i+1))); 1346 i += 2; 1347 } 1348 } 1349 if (res && nch - i != 0) { 1350 res = 0; 1351 err_string("illegal formation for varargslist"); 1352 } 1353 } 1354 } 1355 return res; 1356 } 1357 1358 1359 /* list_iter: list_for | list_if 1360 */ 1361 static int 1362 validate_list_iter(node *tree) 1363 { 1364 int res = (validate_ntype(tree, list_iter) 1365 && validate_numnodes(tree, 1, "list_iter")); 1366 if (res && TYPE(CHILD(tree, 0)) == list_for) 1367 res = validate_list_for(CHILD(tree, 0)); 1368 else 1369 res = validate_list_if(CHILD(tree, 0)); 1370 1371 return res; 1372 } 1373 1374 /* comp_iter: comp_for | comp_if 1375 */ 1376 static int 1377 validate_comp_iter(node *tree) 1378 { 1379 int res = (validate_ntype(tree, comp_iter) 1380 && validate_numnodes(tree, 1, "comp_iter")); 1381 if (res && TYPE(CHILD(tree, 0)) == comp_for) 1382 res = validate_comp_for(CHILD(tree, 0)); 1383 else 1384 res = validate_comp_if(CHILD(tree, 0)); 1385 1386 return res; 1387 } 1388 1389 /* list_for: 'for' exprlist 'in' testlist [list_iter] 1390 */ 1391 static int 1392 validate_list_for(node *tree) 1393 { 1394 int nch = NCH(tree); 1395 int res; 1396 1397 if (nch == 5) 1398 res = validate_list_iter(CHILD(tree, 4)); 1399 else 1400 res = validate_numnodes(tree, 4, "list_for"); 1401 1402 if (res) 1403 res = (validate_name(CHILD(tree, 0), "for") 1404 && validate_exprlist(CHILD(tree, 1)) 1405 && validate_name(CHILD(tree, 2), "in") 1406 && validate_testlist_safe(CHILD(tree, 3))); 1407 1408 return res; 1409 } 1410 1411 /* comp_for: 'for' exprlist 'in' test [comp_iter] 1412 */ 1413 static int 1414 validate_comp_for(node *tree) 1415 { 1416 int nch = NCH(tree); 1417 int res; 1418 1419 if (nch == 5) 1420 res = validate_comp_iter(CHILD(tree, 4)); 1421 else 1422 res = validate_numnodes(tree, 4, "comp_for"); 1423 1424 if (res) 1425 res = (validate_name(CHILD(tree, 0), "for") 1426 && validate_exprlist(CHILD(tree, 1)) 1427 && validate_name(CHILD(tree, 2), "in") 1428 && validate_or_test(CHILD(tree, 3))); 1429 1430 return res; 1431 } 1432 1433 /* list_if: 'if' old_test [list_iter] 1434 */ 1435 static int 1436 validate_list_if(node *tree) 1437 { 1438 int nch = NCH(tree); 1439 int res; 1440 1441 if (nch == 3) 1442 res = validate_list_iter(CHILD(tree, 2)); 1443 else 1444 res = validate_numnodes(tree, 2, "list_if"); 1445 1446 if (res) 1447 res = (validate_name(CHILD(tree, 0), "if") 1448 && validate_old_test(CHILD(tree, 1))); 1449 1450 return res; 1451 } 1452 1453 /* comp_if: 'if' old_test [comp_iter] 1454 */ 1455 static int 1456 validate_comp_if(node *tree) 1457 { 1458 int nch = NCH(tree); 1459 int res; 1460 1461 if (nch == 3) 1462 res = validate_comp_iter(CHILD(tree, 2)); 1463 else 1464 res = validate_numnodes(tree, 2, "comp_if"); 1465 1466 if (res) 1467 res = (validate_name(CHILD(tree, 0), "if") 1468 && validate_old_test(CHILD(tree, 1))); 1469 1470 return res; 1471 } 1472 1473 /* validate_fpdef() 1474 * 1475 * fpdef: 1476 * NAME 1477 * | '(' fplist ')' 1478 */ 1479 static int 1480 validate_fpdef(node *tree) 1481 { 1482 int nch = NCH(tree); 1483 int res = validate_ntype(tree, fpdef); 1484 1485 if (res) { 1486 if (nch == 1) 1487 res = validate_ntype(CHILD(tree, 0), NAME); 1488 else if (nch == 3) 1489 res = (validate_lparen(CHILD(tree, 0)) 1490 && validate_fplist(CHILD(tree, 1)) 1491 && validate_rparen(CHILD(tree, 2))); 1492 else 1493 res = validate_numnodes(tree, 1, "fpdef"); 1494 } 1495 return (res); 1496 } 1497 1498 1499 static int 1500 validate_fplist(node *tree) 1501 { 1502 return (validate_repeating_list(tree, fplist, 1503 validate_fpdef, "fplist")); 1504 } 1505 1506 1507 /* simple_stmt | compound_stmt 1508 * 1509 */ 1510 static int 1511 validate_stmt(node *tree) 1512 { 1513 int res = (validate_ntype(tree, stmt) 1514 && validate_numnodes(tree, 1, "stmt")); 1515 1516 if (res) { 1517 tree = CHILD(tree, 0); 1518 1519 if (TYPE(tree) == simple_stmt) 1520 res = validate_simple_stmt(tree); 1521 else 1522 res = validate_compound_stmt(tree); 1523 } 1524 return (res); 1525 } 1526 1527 1528 /* small_stmt (';' small_stmt)* [';'] NEWLINE 1529 * 1530 */ 1531 static int 1532 validate_simple_stmt(node *tree) 1533 { 1534 int nch = NCH(tree); 1535 int res = (validate_ntype(tree, simple_stmt) 1536 && (nch >= 2) 1537 && validate_small_stmt(CHILD(tree, 0)) 1538 && validate_newline(CHILD(tree, nch - 1))); 1539 1540 if (nch < 2) 1541 res = validate_numnodes(tree, 2, "simple_stmt"); 1542 --nch; /* forget the NEWLINE */ 1543 if (res && is_even(nch)) 1544 res = validate_semi(CHILD(tree, --nch)); 1545 if (res && (nch > 2)) { 1546 int i; 1547 1548 for (i = 1; res && (i < nch); i += 2) 1549 res = (validate_semi(CHILD(tree, i)) 1550 && validate_small_stmt(CHILD(tree, i + 1))); 1551 } 1552 return (res); 1553 } 1554 1555 1556 static int 1557 validate_small_stmt(node *tree) 1558 { 1559 int nch = NCH(tree); 1560 int res = validate_numnodes(tree, 1, "small_stmt"); 1561 1562 if (res) { 1563 int ntype = TYPE(CHILD(tree, 0)); 1564 1565 if ( (ntype == expr_stmt) 1566 || (ntype == print_stmt) 1567 || (ntype == del_stmt) 1568 || (ntype == pass_stmt) 1569 || (ntype == flow_stmt) 1570 || (ntype == import_stmt) 1571 || (ntype == global_stmt) 1572 || (ntype == assert_stmt) 1573 || (ntype == exec_stmt)) 1574 res = validate_node(CHILD(tree, 0)); 1575 else { 1576 res = 0; 1577 err_string("illegal small_stmt child type"); 1578 } 1579 } 1580 else if (nch == 1) { 1581 res = 0; 1582 PyErr_Format(parser_error, 1583 "Unrecognized child node of small_stmt: %d.", 1584 TYPE(CHILD(tree, 0))); 1585 } 1586 return (res); 1587 } 1588 1589 1590 /* compound_stmt: 1591 * if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated 1592 */ 1593 static int 1594 validate_compound_stmt(node *tree) 1595 { 1596 int res = (validate_ntype(tree, compound_stmt) 1597 && validate_numnodes(tree, 1, "compound_stmt")); 1598 int ntype; 1599 1600 if (!res) 1601 return (0); 1602 1603 tree = CHILD(tree, 0); 1604 ntype = TYPE(tree); 1605 if ( (ntype == if_stmt) 1606 || (ntype == while_stmt) 1607 || (ntype == for_stmt) 1608 || (ntype == try_stmt) 1609 || (ntype == with_stmt) 1610 || (ntype == funcdef) 1611 || (ntype == classdef) 1612 || (ntype == decorated)) 1613 res = validate_node(tree); 1614 else { 1615 res = 0; 1616 PyErr_Format(parser_error, 1617 "Illegal compound statement type: %d.", TYPE(tree)); 1618 } 1619 return (res); 1620 } 1621 1622 static int 1623 validate_yield_or_testlist(node *tree) 1624 { 1625 if (TYPE(tree) == yield_expr) 1626 return validate_yield_expr(tree); 1627 else 1628 return validate_testlist(tree); 1629 } 1630 1631 static int 1632 validate_expr_stmt(node *tree) 1633 { 1634 int j; 1635 int nch = NCH(tree); 1636 int res = (validate_ntype(tree, expr_stmt) 1637 && is_odd(nch) 1638 && validate_testlist(CHILD(tree, 0))); 1639 1640 if (res && nch == 3 1641 && TYPE(CHILD(tree, 1)) == augassign) { 1642 res = validate_numnodes(CHILD(tree, 1), 1, "augassign") 1643 && validate_yield_or_testlist(CHILD(tree, 2)); 1644 1645 if (res) { 1646 char *s = STR(CHILD(CHILD(tree, 1), 0)); 1647 1648 res = (strcmp(s, "+=") == 0 1649 || strcmp(s, "-=") == 0 1650 || strcmp(s, "*=") == 0 1651 || strcmp(s, "/=") == 0 1652 || strcmp(s, "//=") == 0 1653 || strcmp(s, "%=") == 0 1654 || strcmp(s, "&=") == 0 1655 || strcmp(s, "|=") == 0 1656 || strcmp(s, "^=") == 0 1657 || strcmp(s, "<<=") == 0 1658 || strcmp(s, ">>=") == 0 1659 || strcmp(s, "**=") == 0); 1660 if (!res) 1661 err_string("illegal augmented assignment operator"); 1662 } 1663 } 1664 else { 1665 for (j = 1; res && (j < nch); j += 2) 1666 res = validate_equal(CHILD(tree, j)) 1667 && validate_yield_or_testlist(CHILD(tree, j + 1)); 1668 } 1669 return (res); 1670 } 1671 1672 1673 /* print_stmt: 1674 * 1675 * 'print' ( [ test (',' test)* [','] ] 1676 * | '>>' test [ (',' test)+ [','] ] ) 1677 */ 1678 static int 1679 validate_print_stmt(node *tree) 1680 { 1681 int nch = NCH(tree); 1682 int res = (validate_ntype(tree, print_stmt) 1683 && (nch > 0) 1684 && validate_name(CHILD(tree, 0), "print")); 1685 1686 if (res && nch > 1) { 1687 int sym = TYPE(CHILD(tree, 1)); 1688 int i = 1; 1689 int allow_trailing_comma = 1; 1690 1691 if (sym == test) 1692 res = validate_test(CHILD(tree, i++)); 1693 else { 1694 if (nch < 3) 1695 res = validate_numnodes(tree, 3, "print_stmt"); 1696 else { 1697 res = (validate_ntype(CHILD(tree, i), RIGHTSHIFT) 1698 && validate_test(CHILD(tree, i+1))); 1699 i += 2; 1700 allow_trailing_comma = 0; 1701 } 1702 } 1703 if (res) { 1704 /* ... (',' test)* [','] */ 1705 while (res && i+2 <= nch) { 1706 res = (validate_comma(CHILD(tree, i)) 1707 && validate_test(CHILD(tree, i+1))); 1708 allow_trailing_comma = 1; 1709 i += 2; 1710 } 1711 if (res && !allow_trailing_comma) 1712 res = validate_numnodes(tree, i, "print_stmt"); 1713 else if (res && i < nch) 1714 res = validate_comma(CHILD(tree, i)); 1715 } 1716 } 1717 return (res); 1718 } 1719 1720 1721 static int 1722 validate_del_stmt(node *tree) 1723 { 1724 return (validate_numnodes(tree, 2, "del_stmt") 1725 && validate_name(CHILD(tree, 0), "del") 1726 && validate_exprlist(CHILD(tree, 1))); 1727 } 1728 1729 1730 static int 1731 validate_return_stmt(node *tree) 1732 { 1733 int nch = NCH(tree); 1734 int res = (validate_ntype(tree, return_stmt) 1735 && ((nch == 1) || (nch == 2)) 1736 && validate_name(CHILD(tree, 0), "return")); 1737 1738 if (res && (nch == 2)) 1739 res = validate_testlist(CHILD(tree, 1)); 1740 1741 return (res); 1742 } 1743 1744 1745 static int 1746 validate_raise_stmt(node *tree) 1747 { 1748 int nch = NCH(tree); 1749 int res = (validate_ntype(tree, raise_stmt) 1750 && ((nch == 1) || (nch == 2) || (nch == 4) || (nch == 6))); 1751 1752 if (res) { 1753 res = validate_name(CHILD(tree, 0), "raise"); 1754 if (res && (nch >= 2)) 1755 res = validate_test(CHILD(tree, 1)); 1756 if (res && nch > 2) { 1757 res = (validate_comma(CHILD(tree, 2)) 1758 && validate_test(CHILD(tree, 3))); 1759 if (res && (nch > 4)) 1760 res = (validate_comma(CHILD(tree, 4)) 1761 && validate_test(CHILD(tree, 5))); 1762 } 1763 } 1764 else 1765 (void) validate_numnodes(tree, 2, "raise"); 1766 if (res && (nch == 4)) 1767 res = (validate_comma(CHILD(tree, 2)) 1768 && validate_test(CHILD(tree, 3))); 1769 1770 return (res); 1771 } 1772 1773 1774 /* yield_expr: 'yield' [testlist] 1775 */ 1776 static int 1777 validate_yield_expr(node *tree) 1778 { 1779 int nch = NCH(tree); 1780 int res = (validate_ntype(tree, yield_expr) 1781 && ((nch == 1) || (nch == 2)) 1782 && validate_name(CHILD(tree, 0), "yield")); 1783 1784 if (res && (nch == 2)) 1785 res = validate_testlist(CHILD(tree, 1)); 1786 1787 return (res); 1788 } 1789 1790 1791 /* yield_stmt: yield_expr 1792 */ 1793 static int 1794 validate_yield_stmt(node *tree) 1795 { 1796 return (validate_ntype(tree, yield_stmt) 1797 && validate_numnodes(tree, 1, "yield_stmt") 1798 && validate_yield_expr(CHILD(tree, 0))); 1799 } 1800 1801 1802 static int 1803 validate_import_as_name(node *tree) 1804 { 1805 int nch = NCH(tree); 1806 int ok = validate_ntype(tree, import_as_name); 1807 1808 if (ok) { 1809 if (nch == 1) 1810 ok = validate_name(CHILD(tree, 0), NULL); 1811 else if (nch == 3) 1812 ok = (validate_name(CHILD(tree, 0), NULL) 1813 && validate_name(CHILD(tree, 1), "as") 1814 && validate_name(CHILD(tree, 2), NULL)); 1815 else 1816 ok = validate_numnodes(tree, 3, "import_as_name"); 1817 } 1818 return ok; 1819 } 1820 1821 1822 /* dotted_name: NAME ("." NAME)* 1823 */ 1824 static int 1825 validate_dotted_name(node *tree) 1826 { 1827 int nch = NCH(tree); 1828 int res = (validate_ntype(tree, dotted_name) 1829 && is_odd(nch) 1830 && validate_name(CHILD(tree, 0), NULL)); 1831 int i; 1832 1833 for (i = 1; res && (i < nch); i += 2) { 1834 res = (validate_dot(CHILD(tree, i)) 1835 && validate_name(CHILD(tree, i+1), NULL)); 1836 } 1837 return res; 1838 } 1839 1840 1841 /* dotted_as_name: dotted_name [NAME NAME] 1842 */ 1843 static int 1844 validate_dotted_as_name(node *tree) 1845 { 1846 int nch = NCH(tree); 1847 int res = validate_ntype(tree, dotted_as_name); 1848 1849 if (res) { 1850 if (nch == 1) 1851 res = validate_dotted_name(CHILD(tree, 0)); 1852 else if (nch == 3) 1853 res = (validate_dotted_name(CHILD(tree, 0)) 1854 && validate_name(CHILD(tree, 1), "as") 1855 && validate_name(CHILD(tree, 2), NULL)); 1856 else { 1857 res = 0; 1858 err_string("illegal number of children for dotted_as_name"); 1859 } 1860 } 1861 return res; 1862 } 1863 1864 1865 /* dotted_as_name (',' dotted_as_name)* */ 1866 static int 1867 validate_dotted_as_names(node *tree) 1868 { 1869 int nch = NCH(tree); 1870 int res = is_odd(nch) && validate_dotted_as_name(CHILD(tree, 0)); 1871 int i; 1872 1873 for (i = 1; res && (i < nch); i += 2) 1874 res = (validate_comma(CHILD(tree, i)) 1875 && validate_dotted_as_name(CHILD(tree, i + 1))); 1876 return (res); 1877 } 1878 1879 1880 /* import_as_name (',' import_as_name)* [','] */ 1881 static int 1882 validate_import_as_names(node *tree) 1883 { 1884 int nch = NCH(tree); 1885 int res = validate_import_as_name(CHILD(tree, 0)); 1886 int i; 1887 1888 for (i = 1; res && (i + 1 < nch); i += 2) 1889 res = (validate_comma(CHILD(tree, i)) 1890 && validate_import_as_name(CHILD(tree, i + 1))); 1891 return (res); 1892 } 1893 1894 1895 /* 'import' dotted_as_names */ 1896 static int 1897 validate_import_name(node *tree) 1898 { 1899 return (validate_ntype(tree, import_name) 1900 && validate_numnodes(tree, 2, "import_name") 1901 && validate_name(CHILD(tree, 0), "import") 1902 && validate_dotted_as_names(CHILD(tree, 1))); 1903 } 1904 1905 /* Helper function to count the number of leading dots in 1906 * 'from ...module import name' 1907 */ 1908 static int 1909 count_from_dots(node *tree) 1910 { 1911 int i; 1912 for (i = 1; i < NCH(tree); i++) 1913 if (TYPE(CHILD(tree, i)) != DOT) 1914 break; 1915 return i-1; 1916 } 1917 1918 /* import_from: ('from' ('.'* dotted_name | '.'+) 1919 * 'import' ('*' | '(' import_as_names ')' | import_as_names)) 1920 */ 1921 static int 1922 validate_import_from(node *tree) 1923 { 1924 int nch = NCH(tree); 1925 int ndots = count_from_dots(tree); 1926 int havename = (TYPE(CHILD(tree, ndots + 1)) == dotted_name); 1927 int offset = ndots + havename; 1928 int res = validate_ntype(tree, import_from) 1929 && (offset >= 1) 1930 && (nch >= 3 + offset) 1931 && validate_name(CHILD(tree, 0), "from") 1932 && (!havename || validate_dotted_name(CHILD(tree, ndots + 1))) 1933 && validate_name(CHILD(tree, offset + 1), "import"); 1934 1935 if (res && TYPE(CHILD(tree, offset + 2)) == LPAR) 1936 res = ((nch == offset + 5) 1937 && validate_lparen(CHILD(tree, offset + 2)) 1938 && validate_import_as_names(CHILD(tree, offset + 3)) 1939 && validate_rparen(CHILD(tree, offset + 4))); 1940 else if (res && TYPE(CHILD(tree, offset + 2)) != STAR) 1941 res = validate_import_as_names(CHILD(tree, offset + 2)); 1942 return (res); 1943 } 1944 1945 1946 /* import_stmt: import_name | import_from */ 1947 static int 1948 validate_import_stmt(node *tree) 1949 { 1950 int nch = NCH(tree); 1951 int res = validate_numnodes(tree, 1, "import_stmt"); 1952 1953 if (res) { 1954 int ntype = TYPE(CHILD(tree, 0)); 1955 1956 if (ntype == import_name || ntype == import_from) 1957 res = validate_node(CHILD(tree, 0)); 1958 else { 1959 res = 0; 1960 err_string("illegal import_stmt child type"); 1961 } 1962 } 1963 else if (nch == 1) { 1964 res = 0; 1965 PyErr_Format(parser_error, 1966 "Unrecognized child node of import_stmt: %d.", 1967 TYPE(CHILD(tree, 0))); 1968 } 1969 return (res); 1970 } 1971 1972 1973 1974 1975 static int 1976 validate_global_stmt(node *tree) 1977 { 1978 int j; 1979 int nch = NCH(tree); 1980 int res = (validate_ntype(tree, global_stmt) 1981 && is_even(nch) && (nch >= 2)); 1982 1983 if (!res && !PyErr_Occurred()) 1984 err_string("illegal global statement"); 1985 1986 if (res) 1987 res = (validate_name(CHILD(tree, 0), "global") 1988 && validate_ntype(CHILD(tree, 1), NAME)); 1989 for (j = 2; res && (j < nch); j += 2) 1990 res = (validate_comma(CHILD(tree, j)) 1991 && validate_ntype(CHILD(tree, j + 1), NAME)); 1992 1993 return (res); 1994 } 1995 1996 1997 /* exec_stmt: 1998 * 1999 * 'exec' expr ['in' test [',' test]] 2000 */ 2001 static int 2002 validate_exec_stmt(node *tree) 2003 { 2004 int nch = NCH(tree); 2005 int res = (validate_ntype(tree, exec_stmt) 2006 && ((nch == 2) || (nch == 4) || (nch == 6)) 2007 && validate_name(CHILD(tree, 0), "exec") 2008 && validate_expr(CHILD(tree, 1))); 2009 2010 if (!res && !PyErr_Occurred()) 2011 err_string("illegal exec statement"); 2012 if (res && (nch > 2)) 2013 res = (validate_name(CHILD(tree, 2), "in") 2014 && validate_test(CHILD(tree, 3))); 2015 if (res && (nch == 6)) 2016 res = (validate_comma(CHILD(tree, 4)) 2017 && validate_test(CHILD(tree, 5))); 2018 2019 return (res); 2020 } 2021 2022 2023 /* assert_stmt: 2024 * 2025 * 'assert' test [',' test] 2026 */ 2027 static int 2028 validate_assert_stmt(node *tree) 2029 { 2030 int nch = NCH(tree); 2031 int res = (validate_ntype(tree, assert_stmt) 2032 && ((nch == 2) || (nch == 4)) 2033 && (validate_name(CHILD(tree, 0), "assert")) 2034 && validate_test(CHILD(tree, 1))); 2035 2036 if (!res && !PyErr_Occurred()) 2037 err_string("illegal assert statement"); 2038 if (res && (nch > 2)) 2039 res = (validate_comma(CHILD(tree, 2)) 2040 && validate_test(CHILD(tree, 3))); 2041 2042 return (res); 2043 } 2044 2045 2046 static int 2047 validate_while(node *tree) 2048 { 2049 int nch = NCH(tree); 2050 int res = (validate_ntype(tree, while_stmt) 2051 && ((nch == 4) || (nch == 7)) 2052 && validate_name(CHILD(tree, 0), "while") 2053 && validate_test(CHILD(tree, 1)) 2054 && validate_colon(CHILD(tree, 2)) 2055 && validate_suite(CHILD(tree, 3))); 2056 2057 if (res && (nch == 7)) 2058 res = (validate_name(CHILD(tree, 4), "else") 2059 && validate_colon(CHILD(tree, 5)) 2060 && validate_suite(CHILD(tree, 6))); 2061 2062 return (res); 2063 } 2064 2065 2066 static int 2067 validate_for(node *tree) 2068 { 2069 int nch = NCH(tree); 2070 int res = (validate_ntype(tree, for_stmt) 2071 && ((nch == 6) || (nch == 9)) 2072 && validate_name(CHILD(tree, 0), "for") 2073 && validate_exprlist(CHILD(tree, 1)) 2074 && validate_name(CHILD(tree, 2), "in") 2075 && validate_testlist(CHILD(tree, 3)) 2076 && validate_colon(CHILD(tree, 4)) 2077 && validate_suite(CHILD(tree, 5))); 2078 2079 if (res && (nch == 9)) 2080 res = (validate_name(CHILD(tree, 6), "else") 2081 && validate_colon(CHILD(tree, 7)) 2082 && validate_suite(CHILD(tree, 8))); 2083 2084 return (res); 2085 } 2086 2087 2088 /* try_stmt: 2089 * 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] 2090 ['finally' ':' suite] 2091 * | 'try' ':' suite 'finally' ':' suite 2092 * 2093 */ 2094 static int 2095 validate_try(node *tree) 2096 { 2097 int nch = NCH(tree); 2098 int pos = 3; 2099 int res = (validate_ntype(tree, try_stmt) 2100 && (nch >= 6) && ((nch % 3) == 0)); 2101 2102 if (res) 2103 res = (validate_name(CHILD(tree, 0), "try") 2104 && validate_colon(CHILD(tree, 1)) 2105 && validate_suite(CHILD(tree, 2)) 2106 && validate_colon(CHILD(tree, nch - 2)) 2107 && validate_suite(CHILD(tree, nch - 1))); 2108 else if (!PyErr_Occurred()) { 2109 const char* name = "except"; 2110 if (TYPE(CHILD(tree, nch - 3)) != except_clause) 2111 name = STR(CHILD(tree, nch - 3)); 2112 2113 PyErr_Format(parser_error, 2114 "Illegal number of children for try/%s node.", name); 2115 } 2116 /* Handle try/finally statement */ 2117 if (res && (TYPE(CHILD(tree, pos)) == NAME) && 2118 (strcmp(STR(CHILD(tree, pos)), "finally") == 0)) { 2119 res = (validate_numnodes(tree, 6, "try/finally") 2120 && validate_colon(CHILD(tree, 4)) 2121 && validate_suite(CHILD(tree, 5))); 2122 return (res); 2123 } 2124 /* try/except statement: skip past except_clause sections */ 2125 while (res && pos < nch && (TYPE(CHILD(tree, pos)) == except_clause)) { 2126 res = (validate_except_clause(CHILD(tree, pos)) 2127 && validate_colon(CHILD(tree, pos + 1)) 2128 && validate_suite(CHILD(tree, pos + 2))); 2129 pos += 3; 2130 } 2131 /* skip else clause */ 2132 if (res && pos < nch && (TYPE(CHILD(tree, pos)) == NAME) && 2133 (strcmp(STR(CHILD(tree, pos)), "else") == 0)) { 2134 res = (validate_colon(CHILD(tree, pos + 1)) 2135 && validate_suite(CHILD(tree, pos + 2))); 2136 pos += 3; 2137 } 2138 if (res && pos < nch) { 2139 /* last clause must be a finally */ 2140 res = (validate_name(CHILD(tree, pos), "finally") 2141 && validate_numnodes(tree, pos + 3, "try/except/finally") 2142 && validate_colon(CHILD(tree, pos + 1)) 2143 && validate_suite(CHILD(tree, pos + 2))); 2144 } 2145 return (res); 2146 } 2147 2148 2149 static int 2150 validate_except_clause(node *tree) 2151 { 2152 int nch = NCH(tree); 2153 int res = (validate_ntype(tree, except_clause) 2154 && ((nch == 1) || (nch == 2) || (nch == 4)) 2155 && validate_name(CHILD(tree, 0), "except")); 2156 2157 if (res && (nch > 1)) 2158 res = validate_test(CHILD(tree, 1)); 2159 if (res && (nch == 4)) { 2160 if (TYPE(CHILD(tree, 2)) == NAME) 2161 res = validate_name(CHILD(tree, 2), "as"); 2162 else 2163 res = validate_comma(CHILD(tree, 2)); 2164 res = res && validate_test(CHILD(tree, 3)); 2165 } 2166 return (res); 2167 } 2168 2169 2170 static int 2171 validate_test(node *tree) 2172 { 2173 int nch = NCH(tree); 2174 int res = validate_ntype(tree, test) && is_odd(nch); 2175 2176 if (res && (TYPE(CHILD(tree, 0)) == lambdef)) 2177 res = ((nch == 1) 2178 && validate_lambdef(CHILD(tree, 0))); 2179 else if (res) { 2180 res = validate_or_test(CHILD(tree, 0)); 2181 res = (res && (nch == 1 || (nch == 5 && 2182 validate_name(CHILD(tree, 1), "if") && 2183 validate_or_test(CHILD(tree, 2)) && 2184 validate_name(CHILD(tree, 3), "else") && 2185 validate_test(CHILD(tree, 4))))); 2186 } 2187 return (res); 2188 } 2189 2190 static int 2191 validate_old_test(node *tree) 2192 { 2193 int nch = NCH(tree); 2194 int res = validate_ntype(tree, old_test) && (nch == 1); 2195 2196 if (res && (TYPE(CHILD(tree, 0)) == old_lambdef)) 2197 res = (validate_old_lambdef(CHILD(tree, 0))); 2198 else if (res) { 2199 res = (validate_or_test(CHILD(tree, 0))); 2200 } 2201 return (res); 2202 } 2203 2204 static int 2205 validate_or_test(node *tree) 2206 { 2207 int nch = NCH(tree); 2208 int res = validate_ntype(tree, or_test) && is_odd(nch); 2209 2210 if (res) { 2211 int pos; 2212 res = validate_and_test(CHILD(tree, 0)); 2213 for (pos = 1; res && (pos < nch); pos += 2) 2214 res = (validate_name(CHILD(tree, pos), "or") 2215 && validate_and_test(CHILD(tree, pos + 1))); 2216 } 2217 return (res); 2218 } 2219 2220 2221 static int 2222 validate_and_test(node *tree) 2223 { 2224 int pos; 2225 int nch = NCH(tree); 2226 int res = (validate_ntype(tree, and_test) 2227 && is_odd(nch) 2228 && validate_not_test(CHILD(tree, 0))); 2229 2230 for (pos = 1; res && (pos < nch); pos += 2) 2231 res = (validate_name(CHILD(tree, pos), "and") 2232 && validate_not_test(CHILD(tree, 0))); 2233 2234 return (res); 2235 } 2236 2237 2238 static int 2239 validate_not_test(node *tree) 2240 { 2241 int nch = NCH(tree); 2242 int res = validate_ntype(tree, not_test) && ((nch == 1) || (nch == 2)); 2243 2244 if (res) { 2245 if (nch == 2) 2246 res = (validate_name(CHILD(tree, 0), "not") 2247 && validate_not_test(CHILD(tree, 1))); 2248 else if (nch == 1) 2249 res = validate_comparison(CHILD(tree, 0)); 2250 } 2251 return (res); 2252 } 2253 2254 2255 static int 2256 validate_comparison(node *tree) 2257 { 2258 int pos; 2259 int nch = NCH(tree); 2260 int res = (validate_ntype(tree, comparison) 2261 && is_odd(nch) 2262 && validate_expr(CHILD(tree, 0))); 2263 2264 for (pos = 1; res && (pos < nch); pos += 2) 2265 res = (validate_comp_op(CHILD(tree, pos)) 2266 && validate_expr(CHILD(tree, pos + 1))); 2267 2268 return (res); 2269 } 2270 2271 2272 static int 2273 validate_comp_op(node *tree) 2274 { 2275 int res = 0; 2276 int nch = NCH(tree); 2277 2278 if (!validate_ntype(tree, comp_op)) 2279 return (0); 2280 if (nch == 1) { 2281 /* 2282 * Only child will be a terminal with a well-defined symbolic name 2283 * or a NAME with a string of either 'is' or 'in' 2284 */ 2285 tree = CHILD(tree, 0); 2286 switch (TYPE(tree)) { 2287 case LESS: 2288 case GREATER: 2289 case EQEQUAL: 2290 case EQUAL: 2291 case LESSEQUAL: 2292 case GREATEREQUAL: 2293 case NOTEQUAL: 2294 res = 1; 2295 break; 2296 case NAME: 2297 res = ((strcmp(STR(tree), "in") == 0) 2298 || (strcmp(STR(tree), "is") == 0)); 2299 if (!res) { 2300 PyErr_Format(parser_error, 2301 "illegal operator '%s'", STR(tree)); 2302 } 2303 break; 2304 default: 2305 err_string("illegal comparison operator type"); 2306 break; 2307 } 2308 } 2309 else if ((res = validate_numnodes(tree, 2, "comp_op")) != 0) { 2310 res = (validate_ntype(CHILD(tree, 0), NAME) 2311 && validate_ntype(CHILD(tree, 1), NAME) 2312 && (((strcmp(STR(CHILD(tree, 0)), "is") == 0) 2313 && (strcmp(STR(CHILD(tree, 1)), "not") == 0)) 2314 || ((strcmp(STR(CHILD(tree, 0)), "not") == 0) 2315 && (strcmp(STR(CHILD(tree, 1)), "in") == 0)))); 2316 if (!res && !PyErr_Occurred()) 2317 err_string("unknown comparison operator"); 2318 } 2319 return (res); 2320 } 2321 2322 2323 static int 2324 validate_expr(node *tree) 2325 { 2326 int j; 2327 int nch = NCH(tree); 2328 int res = (validate_ntype(tree, expr) 2329 && is_odd(nch) 2330 && validate_xor_expr(CHILD(tree, 0))); 2331 2332 for (j = 2; res && (j < nch); j += 2) 2333 res = (validate_xor_expr(CHILD(tree, j)) 2334 && validate_vbar(CHILD(tree, j - 1))); 2335 2336 return (res); 2337 } 2338 2339 2340 static int 2341 validate_xor_expr(node *tree) 2342 { 2343 int j; 2344 int nch = NCH(tree); 2345 int res = (validate_ntype(tree, xor_expr) 2346 && is_odd(nch) 2347 && validate_and_expr(CHILD(tree, 0))); 2348 2349 for (j = 2; res && (j < nch); j += 2) 2350 res = (validate_circumflex(CHILD(tree, j - 1)) 2351 && validate_and_expr(CHILD(tree, j))); 2352 2353 return (res); 2354 } 2355 2356 2357 static int 2358 validate_and_expr(node *tree) 2359 { 2360 int pos; 2361 int nch = NCH(tree); 2362 int res = (validate_ntype(tree, and_expr) 2363 && is_odd(nch) 2364 && validate_shift_expr(CHILD(tree, 0))); 2365 2366 for (pos = 1; res && (pos < nch); pos += 2) 2367 res = (validate_ampersand(CHILD(tree, pos)) 2368 && validate_shift_expr(CHILD(tree, pos + 1))); 2369 2370 return (res); 2371 } 2372 2373 2374 static int 2375 validate_chain_two_ops(node *tree, int (*termvalid)(node *), int op1, int op2) 2376 { 2377 int pos = 1; 2378 int nch = NCH(tree); 2379 int res = (is_odd(nch) 2380 && (*termvalid)(CHILD(tree, 0))); 2381 2382 for ( ; res && (pos < nch); pos += 2) { 2383 if (TYPE(CHILD(tree, pos)) != op1) 2384 res = validate_ntype(CHILD(tree, pos), op2); 2385 if (res) 2386 res = (*termvalid)(CHILD(tree, pos + 1)); 2387 } 2388 return (res); 2389 } 2390 2391 2392 static int 2393 validate_shift_expr(node *tree) 2394 { 2395 return (validate_ntype(tree, shift_expr) 2396 && validate_chain_two_ops(tree, validate_arith_expr, 2397 LEFTSHIFT, RIGHTSHIFT)); 2398 } 2399 2400 2401 static int 2402 validate_arith_expr(node *tree) 2403 { 2404 return (validate_ntype(tree, arith_expr) 2405 && validate_chain_two_ops(tree, validate_term, PLUS, MINUS)); 2406 } 2407 2408 2409 static int 2410 validate_term(node *tree) 2411 { 2412 int pos = 1; 2413 int nch = NCH(tree); 2414 int res = (validate_ntype(tree, term) 2415 && is_odd(nch) 2416 && validate_factor(CHILD(tree, 0))); 2417 2418 for ( ; res && (pos < nch); pos += 2) 2419 res = (((TYPE(CHILD(tree, pos)) == STAR) 2420 || (TYPE(CHILD(tree, pos)) == SLASH) 2421 || (TYPE(CHILD(tree, pos)) == DOUBLESLASH) 2422 || (TYPE(CHILD(tree, pos)) == PERCENT)) 2423 && validate_factor(CHILD(tree, pos + 1))); 2424 2425 return (res); 2426 } 2427 2428 2429 /* factor: 2430 * 2431 * factor: ('+'|'-'|'~') factor | power 2432 */ 2433 static int 2434 validate_factor(node *tree) 2435 { 2436 int nch = NCH(tree); 2437 int res = (validate_ntype(tree, factor) 2438 && (((nch == 2) 2439 && ((TYPE(CHILD(tree, 0)) == PLUS) 2440 || (TYPE(CHILD(tree, 0)) == MINUS) 2441 || (TYPE(CHILD(tree, 0)) == TILDE)) 2442 && validate_factor(CHILD(tree, 1))) 2443 || ((nch == 1) 2444 && validate_power(CHILD(tree, 0))))); 2445 return (res); 2446 } 2447 2448 2449 /* power: 2450 * 2451 * power: atom trailer* ('**' factor)* 2452 */ 2453 static int 2454 validate_power(node *tree) 2455 { 2456 int pos = 1; 2457 int nch = NCH(tree); 2458 int res = (validate_ntype(tree, power) && (nch >= 1) 2459 && validate_atom(CHILD(tree, 0))); 2460 2461 while (res && (pos < nch) && (TYPE(CHILD(tree, pos)) == trailer)) 2462 res = validate_trailer(CHILD(tree, pos++)); 2463 if (res && (pos < nch)) { 2464 if (!is_even(nch - pos)) { 2465 err_string("illegal number of nodes for 'power'"); 2466 return (0); 2467 } 2468 for ( ; res && (pos < (nch - 1)); pos += 2) 2469 res = (validate_doublestar(CHILD(tree, pos)) 2470 && validate_factor(CHILD(tree, pos + 1))); 2471 } 2472 return (res); 2473 } 2474 2475 2476 static int 2477 validate_atom(node *tree) 2478 { 2479 int pos; 2480 int nch = NCH(tree); 2481 int res = validate_ntype(tree, atom); 2482 2483 if (res && nch < 1) 2484 res = validate_numnodes(tree, nch+1, "atom"); 2485 if (res) { 2486 switch (TYPE(CHILD(tree, 0))) { 2487 case LPAR: 2488 res = ((nch <= 3) 2489 && (validate_rparen(CHILD(tree, nch - 1)))); 2490 2491 if (res && (nch == 3)) { 2492 if (TYPE(CHILD(tree, 1))==yield_expr) 2493 res = validate_yield_expr(CHILD(tree, 1)); 2494 else 2495 res = validate_testlist_comp(CHILD(tree, 1)); 2496 } 2497 break; 2498 case LSQB: 2499 if (nch == 2) 2500 res = validate_ntype(CHILD(tree, 1), RSQB); 2501 else if (nch == 3) 2502 res = (validate_listmaker(CHILD(tree, 1)) 2503 && validate_ntype(CHILD(tree, 2), RSQB)); 2504 else { 2505 res = 0; 2506 err_string("illegal list display atom"); 2507 } 2508 break; 2509 case LBRACE: 2510 res = ((nch <= 3) 2511 && validate_ntype(CHILD(tree, nch - 1), RBRACE)); 2512 2513 if (res && (nch == 3)) 2514 res = validate_dictorsetmaker(CHILD(tree, 1)); 2515 break; 2516 case BACKQUOTE: 2517 res = ((nch == 3) 2518 && validate_testlist1(CHILD(tree, 1)) 2519 && validate_ntype(CHILD(tree, 2), BACKQUOTE)); 2520 break; 2521 case NAME: 2522 case NUMBER: 2523 res = (nch == 1); 2524 break; 2525 case STRING: 2526 for (pos = 1; res && (pos < nch); ++pos) 2527 res = validate_ntype(CHILD(tree, pos), STRING); 2528 break; 2529 default: 2530 res = 0; 2531 break; 2532 } 2533 } 2534 return (res); 2535 } 2536 2537 2538 /* listmaker: 2539 * test ( list_for | (',' test)* [','] ) 2540 */ 2541 static int 2542 validate_listmaker(node *tree) 2543 { 2544 int nch = NCH(tree); 2545 int ok = nch; 2546 2547 if (nch == 0) 2548 err_string("missing child nodes of listmaker"); 2549 else 2550 ok = validate_test(CHILD(tree, 0)); 2551 2552 /* 2553 * list_for | (',' test)* [','] 2554 */ 2555 if (nch == 2 && TYPE(CHILD(tree, 1)) == list_for) 2556 ok = validate_list_for(CHILD(tree, 1)); 2557 else { 2558 /* (',' test)* [','] */ 2559 int i = 1; 2560 while (ok && nch - i >= 2) { 2561 ok = (validate_comma(CHILD(tree, i)) 2562 && validate_test(CHILD(tree, i+1))); 2563 i += 2; 2564 } 2565 if (ok && i == nch-1) 2566 ok = validate_comma(CHILD(tree, i)); 2567 else if (i != nch) { 2568 ok = 0; 2569 err_string("illegal trailing nodes for listmaker"); 2570 } 2571 } 2572 return ok; 2573 } 2574 2575 /* testlist_comp: 2576 * test ( comp_for | (',' test)* [','] ) 2577 */ 2578 static int 2579 validate_testlist_comp(node *tree) 2580 { 2581 int nch = NCH(tree); 2582 int ok = nch; 2583 2584 if (nch == 0) 2585 err_string("missing child nodes of testlist_comp"); 2586 else { 2587 ok = validate_test(CHILD(tree, 0)); 2588 } 2589 2590 /* 2591 * comp_for | (',' test)* [','] 2592 */ 2593 if (nch == 2 && TYPE(CHILD(tree, 1)) == comp_for) 2594 ok = validate_comp_for(CHILD(tree, 1)); 2595 else { 2596 /* (',' test)* [','] */ 2597 int i = 1; 2598 while (ok && nch - i >= 2) { 2599 ok = (validate_comma(CHILD(tree, i)) 2600 && validate_test(CHILD(tree, i+1))); 2601 i += 2; 2602 } 2603 if (ok && i == nch-1) 2604 ok = validate_comma(CHILD(tree, i)); 2605 else if (i != nch) { 2606 ok = 0; 2607 err_string("illegal trailing nodes for testlist_comp"); 2608 } 2609 } 2610 return ok; 2611 } 2612 2613 /* decorator: 2614 * '@' dotted_name [ '(' [arglist] ')' ] NEWLINE 2615 */ 2616 static int 2617 validate_decorator(node *tree) 2618 { 2619 int ok; 2620 int nch = NCH(tree); 2621 ok = (validate_ntype(tree, decorator) && 2622 (nch == 3 || nch == 5 || nch == 6) && 2623 validate_at(CHILD(tree, 0)) && 2624 validate_dotted_name(CHILD(tree, 1)) && 2625 validate_newline(RCHILD(tree, -1))); 2626 2627 if (ok && nch != 3) { 2628 ok = (validate_lparen(CHILD(tree, 2)) && 2629 validate_rparen(RCHILD(tree, -2))); 2630 2631 if (ok && nch == 6) 2632 ok = validate_arglist(CHILD(tree, 3)); 2633 } 2634 2635 return ok; 2636 } 2637 2638 /* decorators: 2639 * decorator+ 2640 */ 2641 static int 2642 validate_decorators(node *tree) 2643 { 2644 int i, nch, ok; 2645 nch = NCH(tree); 2646 ok = validate_ntype(tree, decorators) && nch >= 1; 2647 2648 for (i = 0; ok && i < nch; ++i) 2649 ok = validate_decorator(CHILD(tree, i)); 2650 2651 return ok; 2652 } 2653 2654 /* with_item: 2655 * test ['as' expr] 2656 */ 2657 static int 2658 validate_with_item(node *tree) 2659 { 2660 int nch = NCH(tree); 2661 int ok = (validate_ntype(tree, with_item) 2662 && (nch == 1 || nch == 3) 2663 && validate_test(CHILD(tree, 0))); 2664 if (ok && nch == 3) 2665 ok = (validate_name(CHILD(tree, 1), "as") 2666 && validate_expr(CHILD(tree, 2))); 2667 return ok; 2668 } 2669 2670 /* with_stmt: 2671 * 0 1 ... -2 -1 2672 * 'with' with_item (',' with_item)* ':' suite 2673 */ 2674 static int 2675 validate_with_stmt(node *tree) 2676 { 2677 int i; 2678 int nch = NCH(tree); 2679 int ok = (validate_ntype(tree, with_stmt) 2680 && (nch % 2 == 0) 2681 && validate_name(CHILD(tree, 0), "with") 2682 && validate_colon(RCHILD(tree, -2)) 2683 && validate_suite(RCHILD(tree, -1))); 2684 for (i = 1; ok && i < nch - 2; i += 2) 2685 ok = validate_with_item(CHILD(tree, i)); 2686 return ok; 2687 } 2688 2689 /* funcdef: 2690 * 2691 * -5 -4 -3 -2 -1 2692 * 'def' NAME parameters ':' suite 2693 */ 2694 static int 2695 validate_funcdef(node *tree) 2696 { 2697 int nch = NCH(tree); 2698 int ok = (validate_ntype(tree, funcdef) 2699 && (nch == 5) 2700 && validate_name(RCHILD(tree, -5), "def") 2701 && validate_ntype(RCHILD(tree, -4), NAME) 2702 && validate_colon(RCHILD(tree, -2)) 2703 && validate_parameters(RCHILD(tree, -3)) 2704 && validate_suite(RCHILD(tree, -1))); 2705 return ok; 2706 } 2707 2708 2709 /* decorated 2710 * decorators (classdef | funcdef) 2711 */ 2712 static int 2713 validate_decorated(node *tree) 2714 { 2715 int nch = NCH(tree); 2716 int ok = (validate_ntype(tree, decorated) 2717 && (nch == 2) 2718 && validate_decorators(RCHILD(tree, -2))); 2719 if (TYPE(RCHILD(tree, -1)) == funcdef) 2720 ok = ok && validate_funcdef(RCHILD(tree, -1)); 2721 else 2722 ok = ok && validate_class(RCHILD(tree, -1)); 2723 return ok; 2724 } 2725 2726 static int 2727 validate_lambdef(node *tree) 2728 { 2729 int nch = NCH(tree); 2730 int res = (validate_ntype(tree, lambdef) 2731 && ((nch == 3) || (nch == 4)) 2732 && validate_name(CHILD(tree, 0), "lambda") 2733 && validate_colon(CHILD(tree, nch - 2)) 2734 && validate_test(CHILD(tree, nch - 1))); 2735 2736 if (res && (nch == 4)) 2737 res = validate_varargslist(CHILD(tree, 1)); 2738 else if (!res && !PyErr_Occurred()) 2739 (void) validate_numnodes(tree, 3, "lambdef"); 2740 2741 return (res); 2742 } 2743 2744 2745 static int 2746 validate_old_lambdef(node *tree) 2747 { 2748 int nch = NCH(tree); 2749 int res = (validate_ntype(tree, old_lambdef) 2750 && ((nch == 3) || (nch == 4)) 2751 && validate_name(CHILD(tree, 0), "lambda") 2752 && validate_colon(CHILD(tree, nch - 2)) 2753 && validate_test(CHILD(tree, nch - 1))); 2754 2755 if (res && (nch == 4)) 2756 res = validate_varargslist(CHILD(tree, 1)); 2757 else if (!res && !PyErr_Occurred()) 2758 (void) validate_numnodes(tree, 3, "old_lambdef"); 2759 2760 return (res); 2761 } 2762 2763 2764 /* arglist: 2765 * 2766 * (argument ',')* (argument [','] | '*' test [',' '**' test] | '**' test) 2767 */ 2768 static int 2769 validate_arglist(node *tree) 2770 { 2771 int nch = NCH(tree); 2772 int i = 0; 2773 int ok = 1; 2774 2775 if (nch <= 0) 2776 /* raise the right error from having an invalid number of children */ 2777 return validate_numnodes(tree, nch + 1, "arglist"); 2778 2779 if (nch > 1) { 2780 for (i=0; i<nch; i++) { 2781 if (TYPE(CHILD(tree, i)) == argument) { 2782 node *ch = CHILD(tree, i); 2783 if (NCH(ch) == 2 && TYPE(CHILD(ch, 1)) == comp_for) { 2784 err_string("need '(', ')' for generator expression"); 2785 return 0; 2786 } 2787 } 2788 } 2789 } 2790 2791 while (ok && nch-i >= 2) { 2792 /* skip leading (argument ',') */ 2793 ok = (validate_argument(CHILD(tree, i)) 2794 && validate_comma(CHILD(tree, i+1))); 2795 if (ok) 2796 i += 2; 2797 else 2798 PyErr_Clear(); 2799 } 2800 ok = 1; 2801 if (nch-i > 0) { 2802 /* 2803 * argument | '*' test [',' '**' test] | '**' test 2804 */ 2805 int sym = TYPE(CHILD(tree, i)); 2806 2807 if (sym == argument) { 2808 ok = validate_argument(CHILD(tree, i)); 2809 if (ok && i+1 != nch) { 2810 err_string("illegal arglist specification" 2811 " (extra stuff on end)"); 2812 ok = 0; 2813 } 2814 } 2815 else if (sym == STAR) { 2816 ok = validate_star(CHILD(tree, i)); 2817 if (ok && (nch-i == 2)) 2818 ok = validate_test(CHILD(tree, i+1)); 2819 else if (ok && (nch-i == 5)) 2820 ok = (validate_test(CHILD(tree, i+1)) 2821 && validate_comma(CHILD(tree, i+2)) 2822 && validate_doublestar(CHILD(tree, i+3)) 2823 && validate_test(CHILD(tree, i+4))); 2824 else { 2825 err_string("illegal use of '*' in arglist"); 2826 ok = 0; 2827 } 2828 } 2829 else if (sym == DOUBLESTAR) { 2830 if (nch-i == 2) 2831 ok = (validate_doublestar(CHILD(tree, i)) 2832 && validate_test(CHILD(tree, i+1))); 2833 else { 2834 err_string("illegal use of '**' in arglist"); 2835 ok = 0; 2836 } 2837 } 2838 else { 2839 err_string("illegal arglist specification"); 2840 ok = 0; 2841 } 2842 } 2843 return (ok); 2844 } 2845 2846 2847 2848 /* argument: 2849 * 2850 * [test '='] test [comp_for] 2851 */ 2852 static int 2853 validate_argument(node *tree) 2854 { 2855 int nch = NCH(tree); 2856 int res = (validate_ntype(tree, argument) 2857 && ((nch == 1) || (nch == 2) || (nch == 3)) 2858 && validate_test(CHILD(tree, 0))); 2859 2860 if (res && (nch == 2)) 2861 res = validate_comp_for(CHILD(tree, 1)); 2862 else if (res && (nch == 3)) 2863 res = (validate_equal(CHILD(tree, 1)) 2864 && validate_test(CHILD(tree, 2))); 2865 2866 return (res); 2867 } 2868 2869 2870 2871 /* trailer: 2872 * 2873 * '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME 2874 */ 2875 static int 2876 validate_trailer(node *tree) 2877 { 2878 int nch = NCH(tree); 2879 int res = validate_ntype(tree, trailer) && ((nch == 2) || (nch == 3)); 2880 2881 if (res) { 2882 switch (TYPE(CHILD(tree, 0))) { 2883 case LPAR: 2884 res = validate_rparen(CHILD(tree, nch - 1)); 2885 if (res && (nch == 3)) 2886 res = validate_arglist(CHILD(tree, 1)); 2887 break; 2888 case LSQB: 2889 res = (validate_numnodes(tree, 3, "trailer") 2890 && validate_subscriptlist(CHILD(tree, 1)) 2891 && validate_ntype(CHILD(tree, 2), RSQB)); 2892 break; 2893 case DOT: 2894 res = (validate_numnodes(tree, 2, "trailer") 2895 && validate_ntype(CHILD(tree, 1), NAME)); 2896 break; 2897 default: 2898 res = 0; 2899 break; 2900 } 2901 } 2902 else { 2903 (void) validate_numnodes(tree, 2, "trailer"); 2904 } 2905 return (res); 2906 } 2907 2908 2909 /* subscriptlist: 2910 * 2911 * subscript (',' subscript)* [','] 2912 */ 2913 static int 2914 validate_subscriptlist(node *tree) 2915 { 2916 return (validate_repeating_list(tree, subscriptlist, 2917 validate_subscript, "subscriptlist")); 2918 } 2919 2920 2921 /* subscript: 2922 * 2923 * '.' '.' '.' | test | [test] ':' [test] [sliceop] 2924 */ 2925 static int 2926 validate_subscript(node *tree) 2927 { 2928 int offset = 0; 2929 int nch = NCH(tree); 2930 int res = validate_ntype(tree, subscript) && (nch >= 1) && (nch <= 4); 2931 2932 if (!res) { 2933 if (!PyErr_Occurred()) 2934 err_string("invalid number of arguments for subscript node"); 2935 return (0); 2936 } 2937 if (TYPE(CHILD(tree, 0)) == DOT) 2938 /* take care of ('.' '.' '.') possibility */ 2939 return (validate_numnodes(tree, 3, "subscript") 2940 && validate_dot(CHILD(tree, 0)) 2941 && validate_dot(CHILD(tree, 1)) 2942 && validate_dot(CHILD(tree, 2))); 2943 if (nch == 1) { 2944 if (TYPE(CHILD(tree, 0)) == test) 2945 res = validate_test(CHILD(tree, 0)); 2946 else 2947 res = validate_colon(CHILD(tree, 0)); 2948 return (res); 2949 } 2950 /* Must be [test] ':' [test] [sliceop], 2951 * but at least one of the optional components will 2952 * be present, but we don't know which yet. 2953 */ 2954 if ((TYPE(CHILD(tree, 0)) != COLON) || (nch == 4)) { 2955 res = validate_test(CHILD(tree, 0)); 2956 offset = 1; 2957 } 2958 if (res) 2959 res = validate_colon(CHILD(tree, offset)); 2960 if (res) { 2961 int rem = nch - ++offset; 2962 if (rem) { 2963 if (TYPE(CHILD(tree, offset)) == test) { 2964 res = validate_test(CHILD(tree, offset)); 2965 ++offset; 2966 --rem; 2967 } 2968 if (res && rem) 2969 res = validate_sliceop(CHILD(tree, offset)); 2970 } 2971 } 2972 return (res); 2973 } 2974 2975 2976 static int 2977 validate_sliceop(node *tree) 2978 { 2979 int nch = NCH(tree); 2980 int res = ((nch == 1) || validate_numnodes(tree, 2, "sliceop")) 2981 && validate_ntype(tree, sliceop); 2982 if (!res && !PyErr_Occurred()) { 2983 res = validate_numnodes(tree, 1, "sliceop"); 2984 } 2985 if (res) 2986 res = validate_colon(CHILD(tree, 0)); 2987 if (res && (nch == 2)) 2988 res = validate_test(CHILD(tree, 1)); 2989 2990 return (res); 2991 } 2992 2993 2994 static int 2995 validate_exprlist(node *tree) 2996 { 2997 return (validate_repeating_list(tree, exprlist, 2998 validate_expr, "exprlist")); 2999 } 3000 3001 3002 /* 3003 * dictorsetmaker: 3004 * 3005 * (test ':' test (comp_for | (',' test ':' test)* [','])) | 3006 * (test (comp_for | (',' test)* [','])) 3007 */ 3008 static int 3009 validate_dictorsetmaker(node *tree) 3010 { 3011 int nch = NCH(tree); 3012 int ok = validate_ntype(tree, dictorsetmaker); 3013 int i = 0; 3014 int check_trailing_comma = 0; 3015 3016 assert(nch > 0); 3017 3018 if (ok && (nch == 1 || TYPE(CHILD(tree, 1)) == COMMA)) { 3019 /* We got a set: 3020 * test (',' test)* [','] 3021 */ 3022 ok = validate_test(CHILD(tree, i++)); 3023 while (ok && nch - i >= 2) { 3024 ok = (validate_comma(CHILD(tree, i)) 3025 && validate_test(CHILD(tree, i+1))); 3026 i += 2; 3027 } 3028 check_trailing_comma = 1; 3029 } 3030 else if (ok && TYPE(CHILD(tree, 1)) == comp_for) { 3031 /* We got a set comprehension: 3032 * test comp_for 3033 */ 3034 ok = (validate_test(CHILD(tree, 0)) 3035 && validate_comp_for(CHILD(tree, 1))); 3036 } 3037 else if (ok && NCH(tree) > 3 && TYPE(CHILD(tree, 3)) == comp_for) { 3038 /* We got a dict comprehension: 3039 * test ':' test comp_for 3040 */ 3041 ok = (validate_test(CHILD(tree, 0)) 3042 && validate_colon(CHILD(tree, 1)) 3043 && validate_test(CHILD(tree, 2)) 3044 && validate_comp_for(CHILD(tree, 3))); 3045 } 3046 else if (ok) { 3047 /* We got a dict: 3048 * test ':' test (',' test ':' test)* [','] 3049 */ 3050 if (nch >= 3) { 3051 ok = (validate_test(CHILD(tree, i)) 3052 && validate_colon(CHILD(tree, i+1)) 3053 && validate_test(CHILD(tree, i+2))); 3054 i += 3; 3055 } 3056 else { 3057 ok = 0; 3058 err_string("illegal number of nodes for dictorsetmaker"); 3059 } 3060 3061 while (ok && nch - i >= 4) { 3062 ok = (validate_comma(CHILD(tree, i)) 3063 && validate_test(CHILD(tree, i+1)) 3064 && validate_colon(CHILD(tree, i+2)) 3065 && validate_test(CHILD(tree, i+3))); 3066 i += 4; 3067 } 3068 check_trailing_comma = 1; 3069 } 3070 if (ok && check_trailing_comma) { 3071 if (i == nch-1) 3072 ok = validate_comma(CHILD(tree, i)); 3073 else if (i != nch) { 3074 ok = 0; 3075 err_string("illegal trailing nodes for dictorsetmaker"); 3076 } 3077 } 3078 3079 return ok; 3080 } 3081 3082 3083 static int 3084 validate_eval_input(node *tree) 3085 { 3086 int pos; 3087 int nch = NCH(tree); 3088 int res = (validate_ntype(tree, eval_input) 3089 && (nch >= 2) 3090 && validate_testlist(CHILD(tree, 0)) 3091 && validate_ntype(CHILD(tree, nch - 1), ENDMARKER)); 3092 3093 for (pos = 1; res && (pos < (nch - 1)); ++pos) 3094 res = validate_ntype(CHILD(tree, pos), NEWLINE); 3095 3096 return (res); 3097 } 3098 3099 3100 static int 3101 validate_node(node *tree) 3102 { 3103 int nch = 0; /* num. children on current node */ 3104 int res = 1; /* result value */ 3105 node* next = 0; /* node to process after this one */ 3106 3107 while (res && (tree != 0)) { 3108 nch = NCH(tree); 3109 next = 0; 3110 switch (TYPE(tree)) { 3111 /* 3112 * Definition nodes. 3113 */ 3114 case funcdef: 3115 res = validate_funcdef(tree); 3116 break; 3117 case with_stmt: 3118 res = validate_with_stmt(tree); 3119 break; 3120 case classdef: 3121 res = validate_class(tree); 3122 break; 3123 case decorated: 3124 res = validate_decorated(tree); 3125 break; 3126 /* 3127 * "Trivial" parse tree nodes. 3128 * (Why did I call these trivial?) 3129 */ 3130 case stmt: 3131 res = validate_stmt(tree); 3132 break; 3133 case small_stmt: 3134 /* 3135 * expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt 3136 * | import_stmt | global_stmt | exec_stmt | assert_stmt 3137 */ 3138 res = validate_small_stmt(tree); 3139 break; 3140 case flow_stmt: 3141 res = (validate_numnodes(tree, 1, "flow_stmt") 3142 && ((TYPE(CHILD(tree, 0)) == break_stmt) 3143 || (TYPE(CHILD(tree, 0)) == continue_stmt) 3144 || (TYPE(CHILD(tree, 0)) == yield_stmt) 3145 || (TYPE(CHILD(tree, 0)) == return_stmt) 3146 || (TYPE(CHILD(tree, 0)) == raise_stmt))); 3147 if (res) 3148 next = CHILD(tree, 0); 3149 else if (nch == 1) 3150 err_string("illegal flow_stmt type"); 3151 break; 3152 case yield_stmt: 3153 res = validate_yield_stmt(tree); 3154 break; 3155 /* 3156 * Compound statements. 3157 */ 3158 case simple_stmt: 3159 res = validate_simple_stmt(tree); 3160 break; 3161 case compound_stmt: 3162 res = validate_compound_stmt(tree); 3163 break; 3164 /* 3165 * Fundamental statements. 3166 */ 3167 case expr_stmt: 3168 res = validate_expr_stmt(tree); 3169 break; 3170 case print_stmt: 3171 res = validate_print_stmt(tree); 3172 break; 3173 case del_stmt: 3174 res = validate_del_stmt(tree); 3175 break; 3176 case pass_stmt: 3177 res = (validate_numnodes(tree, 1, "pass") 3178 && validate_name(CHILD(tree, 0), "pass")); 3179 break; 3180 case break_stmt: 3181 res = (validate_numnodes(tree, 1, "break") 3182 && validate_name(CHILD(tree, 0), "break")); 3183 break; 3184 case continue_stmt: 3185 res = (validate_numnodes(tree, 1, "continue") 3186 && validate_name(CHILD(tree, 0), "continue")); 3187 break; 3188 case return_stmt: 3189 res = validate_return_stmt(tree); 3190 break; 3191 case raise_stmt: 3192 res = validate_raise_stmt(tree); 3193 break; 3194 case import_stmt: 3195 res = validate_import_stmt(tree); 3196 break; 3197 case import_name: 3198 res = validate_import_name(tree); 3199 break; 3200 case import_from: 3201 res = validate_import_from(tree); 3202 break; 3203 case global_stmt: 3204 res = validate_global_stmt(tree); 3205 break; 3206 case exec_stmt: 3207 res = validate_exec_stmt(tree); 3208 break; 3209 case assert_stmt: 3210 res = validate_assert_stmt(tree); 3211 break; 3212 case if_stmt: 3213 res = validate_if(tree); 3214 break; 3215 case while_stmt: 3216 res = validate_while(tree); 3217 break; 3218 case for_stmt: 3219 res = validate_for(tree); 3220 break; 3221 case try_stmt: 3222 res = validate_try(tree); 3223 break; 3224 case suite: 3225 res = validate_suite(tree); 3226 break; 3227 /* 3228 * Expression nodes. 3229 */ 3230 case testlist: 3231 res = validate_testlist(tree); 3232 break; 3233 case yield_expr: 3234 res = validate_yield_expr(tree); 3235 break; 3236 case testlist1: 3237 res = validate_testlist1(tree); 3238 break; 3239 case test: 3240 res = validate_test(tree); 3241 break; 3242 case and_test: 3243 res = validate_and_test(tree); 3244 break; 3245 case not_test: 3246 res = validate_not_test(tree); 3247 break; 3248 case comparison: 3249 res = validate_comparison(tree); 3250 break; 3251 case exprlist: 3252 res = validate_exprlist(tree); 3253 break; 3254 case comp_op: 3255 res = validate_comp_op(tree); 3256 break; 3257 case expr: 3258 res = validate_expr(tree); 3259 break; 3260 case xor_expr: 3261 res = validate_xor_expr(tree); 3262 break; 3263 case and_expr: 3264 res = validate_and_expr(tree); 3265 break; 3266 case shift_expr: 3267 res = validate_shift_expr(tree); 3268 break; 3269 case arith_expr: 3270 res = validate_arith_expr(tree); 3271 break; 3272 case term: 3273 res = validate_term(tree); 3274 break; 3275 case factor: 3276 res = validate_factor(tree); 3277 break; 3278 case power: 3279 res = validate_power(tree); 3280 break; 3281 case atom: 3282 res = validate_atom(tree); 3283 break; 3284 3285 default: 3286 /* Hopefully never reached! */ 3287 err_string("unrecognized node type"); 3288 res = 0; 3289 break; 3290 } 3291 tree = next; 3292 } 3293 return (res); 3294 } 3295 3296 3297 static int 3298 validate_expr_tree(node *tree) 3299 { 3300 int res = validate_eval_input(tree); 3301 3302 if (!res && !PyErr_Occurred()) 3303 err_string("could not validate expression tuple"); 3304 3305 return (res); 3306 } 3307 3308 3309 /* file_input: 3310 * (NEWLINE | stmt)* ENDMARKER 3311 */ 3312 static int 3313 validate_file_input(node *tree) 3314 { 3315 int j; 3316 int nch = NCH(tree) - 1; 3317 int res = ((nch >= 0) 3318 && validate_ntype(CHILD(tree, nch), ENDMARKER)); 3319 3320 for (j = 0; res && (j < nch); ++j) { 3321 if (TYPE(CHILD(tree, j)) == stmt) 3322 res = validate_stmt(CHILD(tree, j)); 3323 else 3324 res = validate_newline(CHILD(tree, j)); 3325 } 3326 /* This stays in to prevent any internal failures from getting to the 3327 * user. Hopefully, this won't be needed. If a user reports getting 3328 * this, we have some debugging to do. 3329 */ 3330 if (!res && !PyErr_Occurred()) 3331 err_string("VALIDATION FAILURE: report this to the maintainer!"); 3332 3333 return (res); 3334 } 3335 3336 static int 3337 validate_encoding_decl(node *tree) 3338 { 3339 int nch = NCH(tree); 3340 int res = ((nch == 1) 3341 && validate_file_input(CHILD(tree, 0))); 3342 3343 if (!res && !PyErr_Occurred()) 3344 err_string("Error Parsing encoding_decl"); 3345 3346 return res; 3347 } 3348 3349 static PyObject* 3350 pickle_constructor = NULL; 3351 3352 3353 static PyObject* 3354 parser__pickler(PyObject *self, PyObject *args) 3355 { 3356 NOTE(ARGUNUSED(self)) 3357 PyObject *result = NULL; 3358 PyObject *st = NULL; 3359 PyObject *empty_dict = NULL; 3360 3361 if (PyArg_ParseTuple(args, "O!:_pickler", &PyST_Type, &st)) { 3362 PyObject *newargs; 3363 PyObject *tuple; 3364 3365 if ((empty_dict = PyDict_New()) == NULL) 3366 goto finally; 3367 if ((newargs = Py_BuildValue("Oi", st, 1)) == NULL) 3368 goto finally; 3369 tuple = parser_st2tuple((PyST_Object*)NULL, newargs, empty_dict); 3370 if (tuple != NULL) { 3371 result = Py_BuildValue("O(O)", pickle_constructor, tuple); 3372 Py_DECREF(tuple); 3373 } 3374 Py_DECREF(empty_dict); 3375 Py_DECREF(newargs); 3376 } 3377 finally: 3378 Py_XDECREF(empty_dict); 3379 3380 return (result); 3381 } 3382 3383 3384 /* Functions exported by this module. Most of this should probably 3385 * be converted into an ST object with methods, but that is better 3386 * done directly in Python, allowing subclasses to be created directly. 3387 * We'd really have to write a wrapper around it all anyway to allow 3388 * inheritance. 3389 */ 3390 static PyMethodDef parser_functions[] = { 3391 {"ast2tuple", (PyCFunction)parser_ast2tuple, PUBLIC_METHOD_TYPE, 3392 PyDoc_STR("Creates a tuple-tree representation of an ST.")}, 3393 {"ast2list", (PyCFunction)parser_ast2list, PUBLIC_METHOD_TYPE, 3394 PyDoc_STR("Creates a list-tree representation of an ST.")}, 3395 {"compileast", (PyCFunction)parser_compileast,PUBLIC_METHOD_TYPE, 3396 PyDoc_STR("Compiles an ST object into a code object.")}, 3397 {"compilest", (PyCFunction)parser_compilest, PUBLIC_METHOD_TYPE, 3398 PyDoc_STR("Compiles an ST object into a code object.")}, 3399 {"expr", (PyCFunction)parser_expr, PUBLIC_METHOD_TYPE, 3400 PyDoc_STR("Creates an ST object from an expression.")}, 3401 {"isexpr", (PyCFunction)parser_isexpr, PUBLIC_METHOD_TYPE, 3402 PyDoc_STR("Determines if an ST object was created from an expression.")}, 3403 {"issuite", (PyCFunction)parser_issuite, PUBLIC_METHOD_TYPE, 3404 PyDoc_STR("Determines if an ST object was created from a suite.")}, 3405 {"suite", (PyCFunction)parser_suite, PUBLIC_METHOD_TYPE, 3406 PyDoc_STR("Creates an ST object from a suite.")}, 3407 {"sequence2ast", (PyCFunction)parser_tuple2ast, PUBLIC_METHOD_TYPE, 3408 PyDoc_STR("Creates an ST object from a tree representation.")}, 3409 {"sequence2st", (PyCFunction)parser_tuple2st, PUBLIC_METHOD_TYPE, 3410 PyDoc_STR("Creates an ST object from a tree representation.")}, 3411 {"st2tuple", (PyCFunction)parser_st2tuple, PUBLIC_METHOD_TYPE, 3412 PyDoc_STR("Creates a tuple-tree representation of an ST.")}, 3413 {"st2list", (PyCFunction)parser_st2list, PUBLIC_METHOD_TYPE, 3414 PyDoc_STR("Creates a list-tree representation of an ST.")}, 3415 {"tuple2ast", (PyCFunction)parser_tuple2ast, PUBLIC_METHOD_TYPE, 3416 PyDoc_STR("Creates an ST object from a tree representation.")}, 3417 {"tuple2st", (PyCFunction)parser_tuple2st, PUBLIC_METHOD_TYPE, 3418 PyDoc_STR("Creates an ST object from a tree representation.")}, 3419 3420 /* private stuff: support pickle module */ 3421 {"_pickler", (PyCFunction)parser__pickler, METH_VARARGS, 3422 PyDoc_STR("Returns the pickle magic to allow ST objects to be pickled.")}, 3423 3424 {NULL, NULL, 0, NULL} 3425 }; 3426 3427 3428 PyMODINIT_FUNC initparser(void); /* supply a prototype */ 3429 3430 PyMODINIT_FUNC 3431 initparser(void) 3432 { 3433 PyObject *module, *copyreg; 3434 3435 Py_TYPE(&PyST_Type) = &PyType_Type; 3436 module = Py_InitModule("parser", parser_functions); 3437 if (module == NULL) 3438 return; 3439 3440 if (parser_error == 0) 3441 parser_error = PyErr_NewException("parser.ParserError", NULL, NULL); 3442 3443 if (parser_error == 0) 3444 /* caller will check PyErr_Occurred() */ 3445 return; 3446 /* CAUTION: The code next used to skip bumping the refcount on 3447 * parser_error. That's a disaster if initparser() gets called more 3448 * than once. By incref'ing, we ensure that each module dict that 3449 * gets created owns its reference to the shared parser_error object, 3450 * and the file static parser_error vrbl owns a reference too. 3451 */ 3452 Py_INCREF(parser_error); 3453 if (PyModule_AddObject(module, "ParserError", parser_error) != 0) 3454 return; 3455 3456 Py_INCREF(&PyST_Type); 3457 PyModule_AddObject(module, "ASTType", (PyObject*)&PyST_Type); 3458 Py_INCREF(&PyST_Type); 3459 PyModule_AddObject(module, "STType", (PyObject*)&PyST_Type); 3460 3461 PyModule_AddStringConstant(module, "__copyright__", 3462 parser_copyright_string); 3463 PyModule_AddStringConstant(module, "__doc__", 3464 parser_doc_string); 3465 PyModule_AddStringConstant(module, "__version__", 3466 parser_version_string); 3467 3468 /* Register to support pickling. 3469 * If this fails, the import of this module will fail because an 3470 * exception will be raised here; should we clear the exception? 3471 */ 3472 copyreg = PyImport_ImportModuleNoBlock("copy_reg"); 3473 if (copyreg != NULL) { 3474 PyObject *func, *pickler; 3475 3476 func = PyObject_GetAttrString(copyreg, "pickle"); 3477 pickle_constructor = PyObject_GetAttrString(module, "sequence2st"); 3478 pickler = PyObject_GetAttrString(module, "_pickler"); 3479 Py_XINCREF(pickle_constructor); 3480 if ((func != NULL) && (pickle_constructor != NULL) 3481 && (pickler != NULL)) { 3482 PyObject *res; 3483 3484 res = PyObject_CallFunctionObjArgs(func, &PyST_Type, pickler, 3485 pickle_constructor, NULL); 3486 Py_XDECREF(res); 3487 } 3488 Py_XDECREF(func); 3489 Py_XDECREF(pickle_constructor); 3490 Py_XDECREF(pickler); 3491 Py_DECREF(copyreg); 3492 } 3493 } 3494
