1 /* parsermodule.c 2 * 3 * Copyright 1995-1996 by Fred L. Drake, Jr. and Virginia Polytechnic 4 * Institute and State University, Blacksburg, Virginia, USA. 5 * Portions copyright 1991-1995 by Stichting Mathematisch Centrum, 6 * Amsterdam, The Netherlands. Copying is permitted under the terms 7 * associated with the main Python distribution, with the additional 8 * restriction that this additional notice be included and maintained 9 * on all distributed copies. 10 * 11 * This module serves to replace the original parser module written 12 * by Guido. The functionality is not matched precisely, but the 13 * original may be implemented on top of this. This is desirable 14 * since the source of the text to be parsed is now divorced from 15 * this interface. 16 * 17 * Unlike the prior interface, the ability to give a parse tree 18 * produced by Python code as a tuple to the compiler is enabled by 19 * this module. See the documentation for more details. 20 * 21 * I've added some annotations that help with the lint code-checking 22 * program, but they're not complete by a long shot. The real errors 23 * that lint detects are gone, but there are still warnings with 24 * Py_[X]DECREF() and Py_[X]INCREF() macros. The lint annotations 25 * look like "NOTE(...)". 26 */ 27 28 #include "Python.h" /* general Python API */ 29 #include "Python-ast.h" /* mod_ty */ 30 #include "graminit.h" /* symbols defined in the grammar */ 31 #include "node.h" /* internal parser structure */ 32 #include "errcode.h" /* error codes for PyNode_*() */ 33 #include "token.h" /* token definitions */ 34 #include "grammar.h" 35 #include "parsetok.h" 36 /* ISTERMINAL() / ISNONTERMINAL() */ 37 #include "compile.h" 38 #undef Yield 39 #include "ast.h" 40 #include "pyarena.h" 41 42 extern grammar _PyParser_Grammar; /* From graminit.c */ 43 44 #ifdef lint 45 #include <note.h> 46 #else 47 #define NOTE(x) 48 #endif 49 50 /* String constants used to initialize module attributes. 51 * 52 */ 53 static char parser_copyright_string[] = 54 "Copyright 1995-1996 by Virginia Polytechnic Institute & State\n\ 55 University, Blacksburg, Virginia, USA, and Fred L. Drake, Jr., Reston,\n\ 56 Virginia, USA. Portions copyright 1991-1995 by Stichting Mathematisch\n\ 57 Centrum, Amsterdam, The Netherlands."; 58 59 60 PyDoc_STRVAR(parser_doc_string, 61 "This is an interface to Python's internal parser."); 62 63 static char parser_version_string[] = "0.5"; 64 65 66 typedef PyObject* (*SeqMaker) (Py_ssize_t length); 67 typedef int (*SeqInserter) (PyObject* sequence, 68 Py_ssize_t index, 69 PyObject* element); 70 71 /* The function below is copyrighted by Stichting Mathematisch Centrum. The 72 * original copyright statement is included below, and continues to apply 73 * in full to the function immediately following. All other material is 74 * original, copyrighted by Fred L. Drake, Jr. and Virginia Polytechnic 75 * Institute and State University. Changes were made to comply with the 76 * new naming conventions. Added arguments to provide support for creating 77 * lists as well as tuples, and optionally including the line numbers. 78 */ 79 80 81 static PyObject* 82 node2tuple(node *n, /* node to convert */ 83 SeqMaker mkseq, /* create sequence */ 84 SeqInserter addelem, /* func. to add elem. in seq. */ 85 int lineno, /* include line numbers? */ 86 int col_offset) /* include column offsets? */ 87 { 88 if (n == NULL) { 89 Py_INCREF(Py_None); 90 return (Py_None); 91 } 92 if (ISNONTERMINAL(TYPE(n))) { 93 int i; 94 PyObject *v; 95 PyObject *w; 96 97 v = mkseq(1 + NCH(n) + (TYPE(n) == encoding_decl)); 98 if (v == NULL) 99 return (v); 100 w = PyInt_FromLong(TYPE(n)); 101 if (w == NULL) { 102 Py_DECREF(v); 103 return ((PyObject*) NULL); 104 } 105 (void) addelem(v, 0, w); 106 for (i = 0; i < NCH(n); i++) { 107 w = node2tuple(CHILD(n, i), mkseq, addelem, lineno, col_offset); 108 if (w == NULL) { 109 Py_DECREF(v); 110 return ((PyObject*) NULL); 111 } 112 (void) addelem(v, i+1, w); 113 } 114 115 if (TYPE(n) == encoding_decl) 116 (void) addelem(v, i+1, PyString_FromString(STR(n))); 117 return (v); 118 } 119 else if (ISTERMINAL(TYPE(n))) { 120 PyObject *result = mkseq(2 + lineno + col_offset); 121 if (result != NULL) { 122 (void) addelem(result, 0, PyInt_FromLong(TYPE(n))); 123 (void) addelem(result, 1, PyString_FromString(STR(n))); 124 if (lineno) 125 (void) addelem(result, 2, PyInt_FromLong(n->n_lineno)); 126 if (col_offset) 127 (void) addelem(result, 2 + lineno, PyInt_FromLong(n->n_col_offset)); 128 } 129 return (result); 130 } 131 else { 132 PyErr_SetString(PyExc_SystemError, 133 "unrecognized parse tree node type"); 134 return ((PyObject*) NULL); 135 } 136 } 137 /* 138 * End of material copyrighted by Stichting Mathematisch Centrum. 139 */ 140 141 142 143 /* There are two types of intermediate objects we're interested in: 144 * 'eval' and 'exec' types. These constants can be used in the st_type 145 * field of the object type to identify which any given object represents. 146 * These should probably go in an external header to allow other extensions 147 * to use them, but then, we really should be using C++ too. ;-) 148 */ 149 150 #define PyST_EXPR 1 151 #define PyST_SUITE 2 152 153 154 /* These are the internal objects and definitions required to implement the 155 * ST type. Most of the internal names are more reminiscent of the 'old' 156 * naming style, but the code uses the new naming convention. 157 */ 158 159 static PyObject* 160 parser_error = 0; 161 162 163 typedef struct { 164 PyObject_HEAD /* standard object header */ 165 node* st_node; /* the node* returned by the parser */ 166 int st_type; /* EXPR or SUITE ? */ 167 PyCompilerFlags st_flags; /* Parser and compiler flags */ 168 } PyST_Object; 169 170 171 static void parser_free(PyST_Object *st); 172 static PyObject* parser_sizeof(PyST_Object *, void *); 173 static int parser_compare(PyST_Object *left, PyST_Object *right); 174 static PyObject *parser_getattr(PyObject *self, char *name); 175 static PyObject* parser_compilest(PyST_Object *, PyObject *, PyObject *); 176 static PyObject* parser_isexpr(PyST_Object *, PyObject *, PyObject *); 177 static PyObject* parser_issuite(PyST_Object *, PyObject *, PyObject *); 178 static PyObject* parser_st2list(PyST_Object *, PyObject *, PyObject *); 179 static PyObject* parser_st2tuple(PyST_Object *, PyObject *, PyObject *); 180 181 #define PUBLIC_METHOD_TYPE (METH_VARARGS|METH_KEYWORDS) 182 183 static PyMethodDef 184 parser_methods[] = { 185 {"compile", (PyCFunction)parser_compilest, PUBLIC_METHOD_TYPE, 186 PyDoc_STR("Compile this ST object into a code object.")}, 187 {"isexpr", (PyCFunction)parser_isexpr, PUBLIC_METHOD_TYPE, 188 PyDoc_STR("Determines if this ST object was created from an expression.")}, 189 {"issuite", (PyCFunction)parser_issuite, PUBLIC_METHOD_TYPE, 190 PyDoc_STR("Determines if this ST object was created from a suite.")}, 191 {"tolist", (PyCFunction)parser_st2list, PUBLIC_METHOD_TYPE, 192 PyDoc_STR("Creates a list-tree representation of this ST.")}, 193 {"totuple", (PyCFunction)parser_st2tuple, PUBLIC_METHOD_TYPE, 194 PyDoc_STR("Creates a tuple-tree representation of this ST.")}, 195 {"__sizeof__", (PyCFunction)parser_sizeof, METH_NOARGS, 196 PyDoc_STR("Returns size in memory, in bytes.")}, 197 {NULL, NULL, 0, NULL} 198 }; 199 200 static 201 PyTypeObject PyST_Type = { 202 PyVarObject_HEAD_INIT(NULL, 0) 203 "parser.st", /* tp_name */ 204 (int) sizeof(PyST_Object), /* tp_basicsize */ 205 0, /* tp_itemsize */ 206 (destructor)parser_free, /* tp_dealloc */ 207 0, /* tp_print */ 208 parser_getattr, /* tp_getattr */ 209 0, /* tp_setattr */ 210 (cmpfunc)parser_compare, /* tp_compare */ 211 0, /* tp_repr */ 212 0, /* tp_as_number */ 213 0, /* tp_as_sequence */ 214 0, /* tp_as_mapping */ 215 0, /* tp_hash */ 216 0, /* tp_call */ 217 0, /* tp_str */ 218 0, /* tp_getattro */ 219 0, /* tp_setattro */ 220 221 /* Functions to access object as input/output buffer */ 222 0, /* tp_as_buffer */ 223 224 Py_TPFLAGS_DEFAULT, /* tp_flags */ 225 226 /* __doc__ */ 227 "Intermediate representation of a Python parse tree.", 228 0, /* tp_traverse */ 229 0, /* tp_clear */ 230 0, /* tp_richcompare */ 231 0, /* tp_weaklistoffset */ 232 0, /* tp_iter */ 233 0, /* tp_iternext */ 234 parser_methods, /* tp_methods */ 235 }; /* PyST_Type */ 236 237 238 static int 239 parser_compare_nodes(node *left, node *right) 240 { 241 int j; 242 243 if (TYPE(left) < TYPE(right)) 244 return (-1); 245 246 if (TYPE(right) < TYPE(left)) 247 return (1); 248 249 if (ISTERMINAL(TYPE(left))) 250 return (strcmp(STR(left), STR(right))); 251 252 if (NCH(left) < NCH(right)) 253 return (-1); 254 255 if (NCH(right) < NCH(left)) 256 return (1); 257 258 for (j = 0; j < NCH(left); ++j) { 259 int v = parser_compare_nodes(CHILD(left, j), CHILD(right, j)); 260 261 if (v != 0) 262 return (v); 263 } 264 return (0); 265 } 266 267 268 /* int parser_compare(PyST_Object* left, PyST_Object* right) 269 * 270 * Comparison function used by the Python operators ==, !=, <, >, <=, >= 271 * This really just wraps a call to parser_compare_nodes() with some easy 272 * checks and protection code. 273 * 274 */ 275 static int 276 parser_compare(PyST_Object *left, PyST_Object *right) 277 { 278 if (left == right) 279 return (0); 280 281 if ((left == 0) || (right == 0)) 282 return (-1); 283 284 return (parser_compare_nodes(left->st_node, right->st_node)); 285 } 286 287 288 /* parser_newstobject(node* st) 289 * 290 * Allocates a new Python object representing an ST. This is simply the 291 * 'wrapper' object that holds a node* and allows it to be passed around in 292 * Python code. 293 * 294 */ 295 static PyObject* 296 parser_newstobject(node *st, int type) 297 { 298 PyST_Object* o = PyObject_New(PyST_Object, &PyST_Type); 299 300 if (o != 0) { 301 o->st_node = st; 302 o->st_type = type; 303 o->st_flags.cf_flags = 0; 304 } 305 else { 306 PyNode_Free(st); 307 } 308 return ((PyObject*)o); 309 } 310 311 312 /* void parser_free(PyST_Object* st) 313 * 314 * This is called by a del statement that reduces the reference count to 0. 315 * 316 */ 317 static void 318 parser_free(PyST_Object *st) 319 { 320 PyNode_Free(st->st_node); 321 PyObject_Del(st); 322 } 323 324 325 /* parser_st2tuple(PyObject* self, PyObject* args, PyObject* kw) 326 * 327 * This provides conversion from a node* to a tuple object that can be 328 * returned to the Python-level caller. The ST object is not modified. 329 * 330 */ 331 static PyObject* 332 parser_st2tuple(PyST_Object *self, PyObject *args, PyObject *kw) 333 { 334 PyObject *line_option = 0; 335 PyObject *col_option = 0; 336 PyObject *res = 0; 337 int ok; 338 339 static char *keywords[] = {"ast", "line_info", "col_info", NULL}; 340 341 if (self == NULL) { 342 ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|OO:st2tuple", keywords, 343 &PyST_Type, &self, &line_option, 344 &col_option); 345 } 346 else 347 ok = PyArg_ParseTupleAndKeywords(args, kw, "|OO:totuple", &keywords[1], 348 &line_option, &col_option); 349 if (ok != 0) { 350 int lineno = 0; 351 int col_offset = 0; 352 if (line_option != NULL) { 353 lineno = PyObject_IsTrue(line_option); 354 if (lineno < 0) 355 return NULL; 356 } 357 if (col_option != NULL) { 358 col_offset = PyObject_IsTrue(col_option); 359 if (col_offset < 0) 360 return NULL; 361 } 362 /* 363 * Convert ST into a tuple representation. Use Guido's function, 364 * since it's known to work already. 365 */ 366 res = node2tuple(((PyST_Object*)self)->st_node, 367 PyTuple_New, PyTuple_SetItem, lineno, col_offset); 368 } 369 return (res); 370 } 371 372 static PyObject* 373 parser_ast2tuple(PyST_Object *self, PyObject *args, PyObject *kw) 374 { 375 if (PyErr_WarnPy3k("ast2tuple is removed in 3.x; use st2tuple", 1) < 0) 376 return NULL; 377 return parser_st2tuple(self, args, kw); 378 } 379 380 381 /* parser_st2list(PyObject* self, PyObject* args, PyObject* kw) 382 * 383 * This provides conversion from a node* to a list object that can be 384 * returned to the Python-level caller. The ST object is not modified. 385 * 386 */ 387 static PyObject* 388 parser_st2list(PyST_Object *self, PyObject *args, PyObject *kw) 389 { 390 PyObject *line_option = 0; 391 PyObject *col_option = 0; 392 PyObject *res = 0; 393 int ok; 394 395 static char *keywords[] = {"ast", "line_info", "col_info", NULL}; 396 397 if (self == NULL) 398 ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|OO:st2list", keywords, 399 &PyST_Type, &self, &line_option, 400 &col_option); 401 else 402 ok = PyArg_ParseTupleAndKeywords(args, kw, "|OO:tolist", &keywords[1], 403 &line_option, &col_option); 404 if (ok) { 405 int lineno = 0; 406 int col_offset = 0; 407 if (line_option != 0) { 408 lineno = PyObject_IsTrue(line_option); 409 if (lineno < 0) 410 return NULL; 411 } 412 if (col_option != 0) { 413 col_offset = PyObject_IsTrue(col_option); 414 if (col_offset < 0) 415 return NULL; 416 } 417 /* 418 * Convert ST into a tuple representation. Use Guido's function, 419 * since it's known to work already. 420 */ 421 res = node2tuple(self->st_node, 422 PyList_New, PyList_SetItem, lineno, col_offset); 423 } 424 return (res); 425 } 426 427 static PyObject* 428 parser_ast2list(PyST_Object *self, PyObject *args, PyObject *kw) 429 { 430 if (PyErr_WarnPy3k("ast2list is removed in 3.x; use st2list", 1) < 0) 431 return NULL; 432 return parser_st2list(self, args, kw); 433 } 434 435 436 /* parser_compilest(PyObject* self, PyObject* args) 437 * 438 * This function creates code objects from the parse tree represented by 439 * the passed-in data object. An optional file name is passed in as well. 440 * 441 */ 442 static PyObject* 443 parser_compilest(PyST_Object *self, PyObject *args, PyObject *kw) 444 { 445 PyObject* res = 0; 446 PyArena* arena; 447 mod_ty mod; 448 char* str = "<syntax-tree>"; 449 int ok; 450 451 static char *keywords[] = {"ast", "filename", NULL}; 452 453 if (self == NULL) 454 ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|s:compilest", keywords, 455 &PyST_Type, &self, &str); 456 else 457 ok = PyArg_ParseTupleAndKeywords(args, kw, "|s:compile", &keywords[1], 458 &str); 459 460 if (ok) { 461 arena = PyArena_New(); 462 if (arena) { 463 mod = PyAST_FromNode(self->st_node, &(self->st_flags), str, arena); 464 if (mod) { 465 res = (PyObject *)PyAST_Compile(mod, str, &(self->st_flags), arena); 466 } 467 PyArena_Free(arena); 468 } 469 } 470 471 return (res); 472 } 473 474 static PyObject* 475 parser_compileast(PyST_Object *self, PyObject *args, PyObject *kw) 476 { 477 if (PyErr_WarnPy3k("compileast is removed in 3.x; use compilest", 1) < 0) 478 return NULL; 479 return parser_compilest(self, args, kw); 480 } 481 482 483 /* PyObject* parser_isexpr(PyObject* self, PyObject* args) 484 * PyObject* parser_issuite(PyObject* self, PyObject* args) 485 * 486 * Checks the passed-in ST object to determine if it is an expression or 487 * a statement suite, respectively. The return is a Python truth value. 488 * 489 */ 490 static PyObject* 491 parser_isexpr(PyST_Object *self, PyObject *args, PyObject *kw) 492 { 493 PyObject* res = 0; 494 int ok; 495 496 static char *keywords[] = {"ast", NULL}; 497 498 if (self == NULL) 499 ok = PyArg_ParseTupleAndKeywords(args, kw, "O!:isexpr", keywords, 500 &PyST_Type, &self); 501 else 502 ok = PyArg_ParseTupleAndKeywords(args, kw, ":isexpr", &keywords[1]); 503 504 if (ok) { 505 /* Check to see if the ST represents an expression or not. */ 506 res = (self->st_type == PyST_EXPR) ? Py_True : Py_False; 507 Py_INCREF(res); 508 } 509 return (res); 510 } 511 512 513 static PyObject* 514 parser_issuite(PyST_Object *self, PyObject *args, PyObject *kw) 515 { 516 PyObject* res = 0; 517 int ok; 518 519 static char *keywords[] = {"ast", NULL}; 520 521 if (self == NULL) 522 ok = PyArg_ParseTupleAndKeywords(args, kw, "O!:issuite", keywords, 523 &PyST_Type, &self); 524 else 525 ok = PyArg_ParseTupleAndKeywords(args, kw, ":issuite", &keywords[1]); 526 527 if (ok) { 528 /* Check to see if the ST represents an expression or not. */ 529 res = (self->st_type == PyST_EXPR) ? Py_False : Py_True; 530 Py_INCREF(res); 531 } 532 return (res); 533 } 534 535 536 static PyObject* 537 parser_getattr(PyObject *self, char *name) 538 { 539 return (Py_FindMethod(parser_methods, self, name)); 540 } 541 542 543 /* err_string(char* message) 544 * 545 * Sets the error string for an exception of type ParserError. 546 * 547 */ 548 static void 549 err_string(char *message) 550 { 551 PyErr_SetString(parser_error, message); 552 } 553 554 555 /* PyObject* parser_do_parse(PyObject* args, int type) 556 * 557 * Internal function to actually execute the parse and return the result if 558 * successful or set an exception if not. 559 * 560 */ 561 static PyObject* 562 parser_do_parse(PyObject *args, PyObject *kw, char *argspec, int type) 563 { 564 char* string = 0; 565 PyObject* res = 0; 566 int flags = 0; 567 perrdetail err; 568 569 static char *keywords[] = {"source", NULL}; 570 571 if (PyArg_ParseTupleAndKeywords(args, kw, argspec, keywords, &string)) { 572 node* n = PyParser_ParseStringFlagsFilenameEx(string, NULL, 573 &_PyParser_Grammar, 574 (type == PyST_EXPR) 575 ? eval_input : file_input, 576 &err, &flags); 577 578 if (n) { 579 res = parser_newstobject(n, type); 580 if (res) 581 ((PyST_Object *)res)->st_flags.cf_flags = flags & PyCF_MASK; 582 } 583 else 584 PyParser_SetError(&err); 585 } 586 return (res); 587 } 588 589 590 /* PyObject* parser_expr(PyObject* self, PyObject* args) 591 * PyObject* parser_suite(PyObject* self, PyObject* args) 592 * 593 * External interfaces to the parser itself. Which is called determines if 594 * the parser attempts to recognize an expression ('eval' form) or statement 595 * suite ('exec' form). The real work is done by parser_do_parse() above. 596 * 597 */ 598 static PyObject* 599 parser_expr(PyST_Object *self, PyObject *args, PyObject *kw) 600 { 601 NOTE(ARGUNUSED(self)) 602 return (parser_do_parse(args, kw, "s:expr", PyST_EXPR)); 603 } 604 605 606 static PyObject* 607 parser_suite(PyST_Object *self, PyObject *args, PyObject *kw) 608 { 609 NOTE(ARGUNUSED(self)) 610 return (parser_do_parse(args, kw, "s:suite", PyST_SUITE)); 611 } 612 613 614 615 /* This is the messy part of the code. Conversion from a tuple to an ST 616 * object requires that the input tuple be valid without having to rely on 617 * catching an exception from the compiler. This is done to allow the 618 * compiler itself to remain fast, since most of its input will come from 619 * the parser directly, and therefore be known to be syntactically correct. 620 * This validation is done to ensure that we don't core dump the compile 621 * phase, returning an exception instead. 622 * 623 * Two aspects can be broken out in this code: creating a node tree from 624 * the tuple passed in, and verifying that it is indeed valid. It may be 625 * advantageous to expand the number of ST types to include funcdefs and 626 * lambdadefs to take advantage of the optimizer, recognizing those STs 627 * here. They are not necessary, and not quite as useful in a raw form. 628 * For now, let's get expressions and suites working reliably. 629 */ 630 631 632 static node* build_node_tree(PyObject *tuple); 633 static int validate_expr_tree(node *tree); 634 static int validate_file_input(node *tree); 635 static int validate_encoding_decl(node *tree); 636 637 /* PyObject* parser_tuple2st(PyObject* self, PyObject* args) 638 * 639 * This is the public function, called from the Python code. It receives a 640 * single tuple object from the caller, and creates an ST object if the 641 * tuple can be validated. It does this by checking the first code of the 642 * tuple, and, if acceptable, builds the internal representation. If this 643 * step succeeds, the internal representation is validated as fully as 644 * possible with the various validate_*() routines defined below. 645 * 646 * This function must be changed if support is to be added for PyST_FRAGMENT 647 * ST objects. 648 * 649 */ 650 static PyObject* 651 parser_tuple2st(PyST_Object *self, PyObject *args, PyObject *kw) 652 { 653 NOTE(ARGUNUSED(self)) 654 PyObject *st = 0; 655 PyObject *tuple; 656 node *tree; 657 658 static char *keywords[] = {"sequence", NULL}; 659 660 if (!PyArg_ParseTupleAndKeywords(args, kw, "O:sequence2st", keywords, 661 &tuple)) 662 return (0); 663 if (!PySequence_Check(tuple)) { 664 PyErr_SetString(PyExc_ValueError, 665 "sequence2st() requires a single sequence argument"); 666 return (0); 667 } 668 /* 669 * Convert the tree to the internal form before checking it. 670 */ 671 tree = build_node_tree(tuple); 672 if (tree != 0) { 673 int start_sym = TYPE(tree); 674 if (start_sym == eval_input) { 675 /* Might be an eval form. */ 676 if (validate_expr_tree(tree)) 677 st = parser_newstobject(tree, PyST_EXPR); 678 else 679 PyNode_Free(tree); 680 } 681 else if (start_sym == file_input) { 682 /* This looks like an exec form so far. */ 683 if (validate_file_input(tree)) 684 st = parser_newstobject(tree, PyST_SUITE); 685 else 686 PyNode_Free(tree); 687 } 688 else if (start_sym == encoding_decl) { 689 /* This looks like an encoding_decl so far. */ 690 if (validate_encoding_decl(tree)) 691 st = parser_newstobject(tree, PyST_SUITE); 692 else 693 PyNode_Free(tree); 694 } 695 else { 696 /* This is a fragment, at best. */ 697 PyNode_Free(tree); 698 err_string("parse tree does not use a valid start symbol"); 699 } 700 } 701 /* Make sure we raise an exception on all errors. We should never 702 * get this, but we'd do well to be sure something is done. 703 */ 704 if (st == NULL && !PyErr_Occurred()) 705 err_string("unspecified ST error occurred"); 706 707 return st; 708 } 709 710 static PyObject* 711 parser_tuple2ast(PyST_Object *self, PyObject *args, PyObject *kw) 712 { 713 if (PyErr_WarnPy3k("tuple2ast is removed in 3.x; use tuple2st", 1) < 0) 714 return NULL; 715 return parser_tuple2st(self, args, kw); 716 } 717 718 static PyObject * 719 parser_sizeof(PyST_Object *st, void *unused) 720 { 721 Py_ssize_t res; 722 723 res = _PyObject_SIZE(Py_TYPE(st)) + _PyNode_SizeOf(st->st_node); 724 return PyLong_FromSsize_t(res); 725 } 726 727 728 /* node* build_node_children() 729 * 730 * Iterate across the children of the current non-terminal node and build 731 * their structures. If successful, return the root of this portion of 732 * the tree, otherwise, 0. Any required exception will be specified already, 733 * and no memory will have been deallocated. 734 * 735 */ 736 static node* 737 build_node_children(PyObject *tuple, node *root, int *line_num) 738 { 739 Py_ssize_t len = PyObject_Size(tuple); 740 Py_ssize_t i; 741 int err; 742 743 if (len < 0) { 744 return NULL; 745 } 746 for (i = 1; i < len; ++i) { 747 /* elem must always be a sequence, however simple */ 748 PyObject* elem = PySequence_GetItem(tuple, i); 749 int ok = elem != NULL; 750 long type = 0; 751 char *strn = 0; 752 753 if (ok) 754 ok = PySequence_Check(elem); 755 if (ok) { 756 PyObject *temp = PySequence_GetItem(elem, 0); 757 if (temp == NULL) 758 ok = 0; 759 else { 760 ok = PyInt_Check(temp); 761 if (ok) 762 type = PyInt_AS_LONG(temp); 763 Py_DECREF(temp); 764 } 765 } 766 if (!ok) { 767 PyObject *err = Py_BuildValue("os", elem, 768 "Illegal node construct."); 769 PyErr_SetObject(parser_error, err); 770 Py_XDECREF(err); 771 Py_XDECREF(elem); 772 return NULL; 773 } 774 if (ISTERMINAL(type)) { 775 Py_ssize_t len = PyObject_Size(elem); 776 PyObject *temp; 777 778 if ((len != 2) && (len != 3)) { 779 err_string("terminal nodes must have 2 or 3 entries"); 780 Py_DECREF(elem); 781 return NULL; 782 } 783 temp = PySequence_GetItem(elem, 1); 784 if (temp == NULL) { 785 Py_DECREF(elem); 786 return NULL; 787 } 788 if (!PyString_Check(temp)) { 789 PyErr_Format(parser_error, 790 "second item in terminal node must be a string," 791 " found %s", 792 Py_TYPE(temp)->tp_name); 793 Py_DECREF(temp); 794 Py_DECREF(elem); 795 return NULL; 796 } 797 if (len == 3) { 798 PyObject *o = PySequence_GetItem(elem, 2); 799 if (o == NULL) { 800 Py_DECREF(temp); 801 Py_DECREF(elem); 802 return NULL; 803 } 804 if (PyInt_Check(o)) 805 *line_num = PyInt_AS_LONG(o); 806 else { 807 PyErr_Format(parser_error, 808 "third item in terminal node must be an" 809 " integer, found %s", 810 Py_TYPE(temp)->tp_name); 811 Py_DECREF(o); 812 Py_DECREF(temp); 813 Py_DECREF(elem); 814 return NULL; 815 } 816 Py_DECREF(o); 817 } 818 len = PyString_GET_SIZE(temp) + 1; 819 strn = (char *)PyObject_MALLOC(len); 820 if (strn == NULL) { 821 Py_DECREF(temp); 822 Py_DECREF(elem); 823 PyErr_NoMemory(); 824 return NULL; 825 } 826 (void) memcpy(strn, PyString_AS_STRING(temp), len); 827 Py_DECREF(temp); 828 } 829 else if (!ISNONTERMINAL(type)) { 830 /* 831 * It has to be one or the other; this is an error. 832 * Raise an exception. 833 */ 834 PyObject *err = Py_BuildValue("Os", elem, "unknown node type."); 835 PyErr_SetObject(parser_error, err); 836 Py_XDECREF(err); 837 Py_DECREF(elem); 838 return NULL; 839 } 840 err = PyNode_AddChild(root, type, strn, *line_num, 0); 841 if (err == E_NOMEM) { 842 Py_DECREF(elem); 843 PyObject_FREE(strn); 844 PyErr_NoMemory(); 845 return NULL; 846 } 847 if (err == E_OVERFLOW) { 848 Py_DECREF(elem); 849 PyObject_FREE(strn); 850 PyErr_SetString(PyExc_ValueError, 851 "unsupported number of child nodes"); 852 return NULL; 853 } 854 855 if (ISNONTERMINAL(type)) { 856 node* new_child = CHILD(root, i - 1); 857 858 if (new_child != build_node_children(elem, new_child, line_num)) { 859 Py_DECREF(elem); 860 return NULL; 861 } 862 } 863 else if (type == NEWLINE) { /* It's true: we increment the */ 864 ++(*line_num); /* line number *after* the newline! */ 865 } 866 Py_DECREF(elem); 867 } 868 return root; 869 } 870 871 872 static node* 873 build_node_tree(PyObject *tuple) 874 { 875 node* res = 0; 876 PyObject *temp = PySequence_GetItem(tuple, 0); 877 long num = -1; 878 879 if (temp != NULL) 880 num = PyInt_AsLong(temp); 881 Py_XDECREF(temp); 882 if (ISTERMINAL(num)) { 883 /* 884 * The tuple is simple, but it doesn't start with a start symbol. 885 * Raise an exception now and be done with it. 886 */ 887 tuple = Py_BuildValue("os", tuple, 888 "Illegal syntax-tree; cannot start with terminal symbol."); 889 PyErr_SetObject(parser_error, tuple); 890 Py_XDECREF(tuple); 891 } 892 else if (ISNONTERMINAL(num)) { 893 /* 894 * Not efficient, but that can be handled later. 895 */ 896 int line_num = 0; 897 PyObject *encoding = NULL; 898 899 if (num == encoding_decl) { 900 encoding = PySequence_GetItem(tuple, 2); 901 if (encoding == NULL) { 902 PyErr_SetString(parser_error, "missed encoding"); 903 return NULL; 904 } 905 if (!PyString_Check(encoding)) { 906 PyErr_Format(parser_error, 907 "encoding must be a string, found %.200s", 908 Py_TYPE(encoding)->tp_name); 909 Py_DECREF(encoding); 910 return NULL; 911 } 912 /* tuple isn't borrowed anymore here, need to DECREF */ 913 tuple = PySequence_GetSlice(tuple, 0, 2); 914 if (tuple == NULL) { 915 Py_DECREF(encoding); 916 return NULL; 917 } 918 } 919 res = PyNode_New(num); 920 if (res != NULL) { 921 if (res != build_node_children(tuple, res, &line_num)) { 922 PyNode_Free(res); 923 res = NULL; 924 } 925 if (res && encoding) { 926 Py_ssize_t len; 927 len = PyString_GET_SIZE(encoding) + 1; 928 res->n_str = (char *)PyObject_MALLOC(len); 929 if (res->n_str == NULL) { 930 PyNode_Free(res); 931 Py_DECREF(encoding); 932 Py_DECREF(tuple); 933 PyErr_NoMemory(); 934 return NULL; 935 } 936 (void) memcpy(res->n_str, PyString_AS_STRING(encoding), len); 937 } 938 } 939 if (encoding != NULL) { 940 Py_DECREF(encoding); 941 Py_DECREF(tuple); 942 } 943 } 944 else { 945 /* The tuple is illegal -- if the number is neither TERMINAL nor 946 * NONTERMINAL, we can't use it. Not sure the implementation 947 * allows this condition, but the API doesn't preclude it. 948 */ 949 PyObject *err = Py_BuildValue("Os", tuple, 950 "Illegal component tuple."); 951 PyErr_SetObject(parser_error, err); 952 Py_XDECREF(err); 953 } 954 955 return (res); 956 } 957 958 959 /* 960 * Validation routines used within the validation section: 961 */ 962 static int validate_terminal(node *terminal, int type, char *string); 963 964 #define validate_ampersand(ch) validate_terminal(ch, AMPER, "&") 965 #define validate_circumflex(ch) validate_terminal(ch, CIRCUMFLEX, "^") 966 #define validate_colon(ch) validate_terminal(ch, COLON, ":") 967 #define validate_comma(ch) validate_terminal(ch, COMMA, ",") 968 #define validate_dedent(ch) validate_terminal(ch, DEDENT, "") 969 #define validate_equal(ch) validate_terminal(ch, EQUAL, "=") 970 #define validate_indent(ch) validate_terminal(ch, INDENT, (char*)NULL) 971 #define validate_lparen(ch) validate_terminal(ch, LPAR, "(") 972 #define validate_newline(ch) validate_terminal(ch, NEWLINE, (char*)NULL) 973 #define validate_rparen(ch) validate_terminal(ch, RPAR, ")") 974 #define validate_semi(ch) validate_terminal(ch, SEMI, ";") 975 #define validate_star(ch) validate_terminal(ch, STAR, "*") 976 #define validate_vbar(ch) validate_terminal(ch, VBAR, "|") 977 #define validate_doublestar(ch) validate_terminal(ch, DOUBLESTAR, "**") 978 #define validate_dot(ch) validate_terminal(ch, DOT, ".") 979 #define validate_at(ch) validate_terminal(ch, AT, "@") 980 #define validate_name(ch, str) validate_terminal(ch, NAME, str) 981 982 #define VALIDATER(n) static int validate_##n(node *tree) 983 984 VALIDATER(node); VALIDATER(small_stmt); 985 VALIDATER(class); VALIDATER(node); 986 VALIDATER(parameters); VALIDATER(suite); 987 VALIDATER(testlist); VALIDATER(varargslist); 988 VALIDATER(fpdef); VALIDATER(fplist); 989 VALIDATER(stmt); VALIDATER(simple_stmt); 990 VALIDATER(expr_stmt); VALIDATER(power); 991 VALIDATER(print_stmt); VALIDATER(del_stmt); 992 VALIDATER(return_stmt); VALIDATER(list_iter); 993 VALIDATER(raise_stmt); VALIDATER(import_stmt); 994 VALIDATER(import_name); VALIDATER(import_from); 995 VALIDATER(global_stmt); VALIDATER(list_if); 996 VALIDATER(assert_stmt); VALIDATER(list_for); 997 VALIDATER(exec_stmt); VALIDATER(compound_stmt); 998 VALIDATER(while); VALIDATER(for); 999 VALIDATER(try); VALIDATER(except_clause); 1000 VALIDATER(test); VALIDATER(and_test); 1001 VALIDATER(not_test); VALIDATER(comparison); 1002 VALIDATER(comp_op); VALIDATER(expr); 1003 VALIDATER(xor_expr); VALIDATER(and_expr); 1004 VALIDATER(shift_expr); VALIDATER(arith_expr); 1005 VALIDATER(term); VALIDATER(factor); 1006 VALIDATER(atom); VALIDATER(lambdef); 1007 VALIDATER(trailer); VALIDATER(subscript); 1008 VALIDATER(subscriptlist); VALIDATER(sliceop); 1009 VALIDATER(exprlist); VALIDATER(dictorsetmaker); 1010 VALIDATER(arglist); VALIDATER(argument); 1011 VALIDATER(listmaker); VALIDATER(yield_stmt); 1012 VALIDATER(testlist1); VALIDATER(comp_for); 1013 VALIDATER(comp_iter); VALIDATER(comp_if); 1014 VALIDATER(testlist_comp); VALIDATER(yield_expr); 1015 VALIDATER(yield_or_testlist); VALIDATER(or_test); 1016 VALIDATER(old_test); VALIDATER(old_lambdef); 1017 1018 #undef VALIDATER 1019 1020 #define is_even(n) (((n) & 1) == 0) 1021 #define is_odd(n) (((n) & 1) == 1) 1022 1023 1024 static int 1025 validate_ntype(node *n, int t) 1026 { 1027 if (TYPE(n) != t) { 1028 PyErr_Format(parser_error, "Expected node type %d, got %d.", 1029 t, TYPE(n)); 1030 return 0; 1031 } 1032 return 1; 1033 } 1034 1035 1036 /* Verifies that the number of child nodes is exactly 'num', raising 1037 * an exception if it isn't. The exception message does not indicate 1038 * the exact number of nodes, allowing this to be used to raise the 1039 * "right" exception when the wrong number of nodes is present in a 1040 * specific variant of a statement's syntax. This is commonly used 1041 * in that fashion. 1042 */ 1043 static int 1044 validate_numnodes(node *n, int num, const char *const name) 1045 { 1046 if (NCH(n) != num) { 1047 PyErr_Format(parser_error, 1048 "Illegal number of children for %s node.", name); 1049 return 0; 1050 } 1051 return 1; 1052 } 1053 1054 1055 static int 1056 validate_terminal(node *terminal, int type, char *string) 1057 { 1058 int res = (validate_ntype(terminal, type) 1059 && ((string == 0) || (strcmp(string, STR(terminal)) == 0))); 1060 1061 if (!res && !PyErr_Occurred()) { 1062 PyErr_Format(parser_error, 1063 "Illegal terminal: expected \"%s\"", string); 1064 } 1065 return (res); 1066 } 1067 1068 1069 /* X (',' X) [','] 1070 */ 1071 static int 1072 validate_repeating_list(node *tree, int ntype, int (*vfunc)(node *), 1073 const char *const name) 1074 { 1075 int nch = NCH(tree); 1076 int res = (nch && validate_ntype(tree, ntype) 1077 && vfunc(CHILD(tree, 0))); 1078 1079 if (!res && !PyErr_Occurred()) 1080 (void) validate_numnodes(tree, 1, name); 1081 else { 1082 if (is_even(nch)) 1083 res = validate_comma(CHILD(tree, --nch)); 1084 if (res && nch > 1) { 1085 int pos = 1; 1086 for ( ; res && pos < nch; pos += 2) 1087 res = (validate_comma(CHILD(tree, pos)) 1088 && vfunc(CHILD(tree, pos + 1))); 1089 } 1090 } 1091 return (res); 1092 } 1093 1094 1095 /* validate_class() 1096 * 1097 * classdef: 1098 * 'class' NAME ['(' testlist ')'] ':' suite 1099 */ 1100 static int 1101 validate_class(node *tree) 1102 { 1103 int nch = NCH(tree); 1104 int res = (validate_ntype(tree, classdef) && 1105 ((nch == 4) || (nch == 6) || (nch == 7))); 1106 1107 if (res) { 1108 res = (validate_name(CHILD(tree, 0), "class") 1109 && validate_ntype(CHILD(tree, 1), NAME) 1110 && validate_colon(CHILD(tree, nch - 2)) 1111 && validate_suite(CHILD(tree, nch - 1))); 1112 } 1113 else { 1114 (void) validate_numnodes(tree, 4, "class"); 1115 } 1116 1117 if (res) { 1118 if (nch == 7) { 1119 res = ((validate_lparen(CHILD(tree, 2)) && 1120 validate_testlist(CHILD(tree, 3)) && 1121 validate_rparen(CHILD(tree, 4)))); 1122 } 1123 else if (nch == 6) { 1124 res = (validate_lparen(CHILD(tree,2)) && 1125 validate_rparen(CHILD(tree,3))); 1126 } 1127 } 1128 return (res); 1129 } 1130 1131 1132 /* if_stmt: 1133 * 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] 1134 */ 1135 static int 1136 validate_if(node *tree) 1137 { 1138 int nch = NCH(tree); 1139 int res = (validate_ntype(tree, if_stmt) 1140 && (nch >= 4) 1141 && validate_name(CHILD(tree, 0), "if") 1142 && validate_test(CHILD(tree, 1)) 1143 && validate_colon(CHILD(tree, 2)) 1144 && validate_suite(CHILD(tree, 3))); 1145 1146 if (res && ((nch % 4) == 3)) { 1147 /* ... 'else' ':' suite */ 1148 res = (validate_name(CHILD(tree, nch - 3), "else") 1149 && validate_colon(CHILD(tree, nch - 2)) 1150 && validate_suite(CHILD(tree, nch - 1))); 1151 nch -= 3; 1152 } 1153 else if (!res && !PyErr_Occurred()) 1154 (void) validate_numnodes(tree, 4, "if"); 1155 if ((nch % 4) != 0) 1156 /* Will catch the case for nch < 4 */ 1157 res = validate_numnodes(tree, 0, "if"); 1158 else if (res && (nch > 4)) { 1159 /* ... ('elif' test ':' suite)+ ... */ 1160 int j = 4; 1161 while ((j < nch) && res) { 1162 res = (validate_name(CHILD(tree, j), "elif") 1163 && validate_colon(CHILD(tree, j + 2)) 1164 && validate_test(CHILD(tree, j + 1)) 1165 && validate_suite(CHILD(tree, j + 3))); 1166 j += 4; 1167 } 1168 } 1169 return (res); 1170 } 1171 1172 1173 /* parameters: 1174 * '(' [varargslist] ')' 1175 * 1176 */ 1177 static int 1178 validate_parameters(node *tree) 1179 { 1180 int nch = NCH(tree); 1181 int res = validate_ntype(tree, parameters) && ((nch == 2) || (nch == 3)); 1182 1183 if (res) { 1184 res = (validate_lparen(CHILD(tree, 0)) 1185 && validate_rparen(CHILD(tree, nch - 1))); 1186 if (res && (nch == 3)) 1187 res = validate_varargslist(CHILD(tree, 1)); 1188 } 1189 else { 1190 (void) validate_numnodes(tree, 2, "parameters"); 1191 } 1192 return (res); 1193 } 1194 1195 1196 /* validate_suite() 1197 * 1198 * suite: 1199 * simple_stmt 1200 * | NEWLINE INDENT stmt+ DEDENT 1201 */ 1202 static int 1203 validate_suite(node *tree) 1204 { 1205 int nch = NCH(tree); 1206 int res = (validate_ntype(tree, suite) && ((nch == 1) || (nch >= 4))); 1207 1208 if (res && (nch == 1)) 1209 res = validate_simple_stmt(CHILD(tree, 0)); 1210 else if (res) { 1211 /* NEWLINE INDENT stmt+ DEDENT */ 1212 res = (validate_newline(CHILD(tree, 0)) 1213 && validate_indent(CHILD(tree, 1)) 1214 && validate_stmt(CHILD(tree, 2)) 1215 && validate_dedent(CHILD(tree, nch - 1))); 1216 1217 if (res && (nch > 4)) { 1218 int i = 3; 1219 --nch; /* forget the DEDENT */ 1220 for ( ; res && (i < nch); ++i) 1221 res = validate_stmt(CHILD(tree, i)); 1222 } 1223 else if (nch < 4) 1224 res = validate_numnodes(tree, 4, "suite"); 1225 } 1226 return (res); 1227 } 1228 1229 1230 static int 1231 validate_testlist(node *tree) 1232 { 1233 return (validate_repeating_list(tree, testlist, 1234 validate_test, "testlist")); 1235 } 1236 1237 1238 static int 1239 validate_testlist1(node *tree) 1240 { 1241 return (validate_repeating_list(tree, testlist1, 1242 validate_test, "testlist1")); 1243 } 1244 1245 1246 static int 1247 validate_testlist_safe(node *tree) 1248 { 1249 return (validate_repeating_list(tree, testlist_safe, 1250 validate_old_test, "testlist_safe")); 1251 } 1252 1253 1254 /* '*' NAME [',' '**' NAME] | '**' NAME 1255 */ 1256 static int 1257 validate_varargslist_trailer(node *tree, int start) 1258 { 1259 int nch = NCH(tree); 1260 int res = 0; 1261 int sym; 1262 1263 if (nch <= start) { 1264 err_string("expected variable argument trailer for varargslist"); 1265 return 0; 1266 } 1267 sym = TYPE(CHILD(tree, start)); 1268 if (sym == STAR) { 1269 /* 1270 * ('*' NAME [',' '**' NAME] 1271 */ 1272 if (nch-start == 2) 1273 res = validate_name(CHILD(tree, start+1), NULL); 1274 else if (nch-start == 5) 1275 res = (validate_name(CHILD(tree, start+1), NULL) 1276 && validate_comma(CHILD(tree, start+2)) 1277 && validate_doublestar(CHILD(tree, start+3)) 1278 && validate_name(CHILD(tree, start+4), NULL)); 1279 } 1280 else if (sym == DOUBLESTAR) { 1281 /* 1282 * '**' NAME 1283 */ 1284 if (nch-start == 2) 1285 res = validate_name(CHILD(tree, start+1), NULL); 1286 } 1287 if (!res) 1288 err_string("illegal variable argument trailer for varargslist"); 1289 return res; 1290 } 1291 1292 1293 /* validate_varargslist() 1294 * 1295 * varargslist: 1296 * (fpdef ['=' test] ',')* 1297 * ('*' NAME [',' '**' NAME] 1298 * | '**' NAME) 1299 * | fpdef ['=' test] (',' fpdef ['=' test])* [','] 1300 * 1301 */ 1302 static int 1303 validate_varargslist(node *tree) 1304 { 1305 int nch = NCH(tree); 1306 int res = validate_ntype(tree, varargslist) && (nch != 0); 1307 int sym; 1308 1309 if (!res) 1310 return 0; 1311 if (nch < 1) { 1312 err_string("varargslist missing child nodes"); 1313 return 0; 1314 } 1315 sym = TYPE(CHILD(tree, 0)); 1316 if (sym == STAR || sym == DOUBLESTAR) 1317 /* whole thing matches: 1318 * '*' NAME [',' '**' NAME] | '**' NAME 1319 */ 1320 res = validate_varargslist_trailer(tree, 0); 1321 else if (sym == fpdef) { 1322 int i = 0; 1323 1324 sym = TYPE(CHILD(tree, nch-1)); 1325 if (sym == NAME) { 1326 /* 1327 * (fpdef ['=' test] ',')+ 1328 * ('*' NAME [',' '**' NAME] 1329 * | '**' NAME) 1330 */ 1331 /* skip over (fpdef ['=' test] ',')+ */ 1332 while (res && (i+2 <= nch)) { 1333 res = validate_fpdef(CHILD(tree, i)); 1334 ++i; 1335 if (res && TYPE(CHILD(tree, i)) == EQUAL && (i+2 <= nch)) { 1336 res = (validate_equal(CHILD(tree, i)) 1337 && validate_test(CHILD(tree, i+1))); 1338 if (res) 1339 i += 2; 1340 } 1341 if (res && i < nch) { 1342 res = validate_comma(CHILD(tree, i)); 1343 ++i; 1344 if (res && i < nch 1345 && (TYPE(CHILD(tree, i)) == DOUBLESTAR 1346 || TYPE(CHILD(tree, i)) == STAR)) 1347 break; 1348 } 1349 } 1350 /* ... '*' NAME [',' '**' NAME] | '**' NAME 1351 * i --^^^ 1352 */ 1353 if (res) 1354 res = validate_varargslist_trailer(tree, i); 1355 } 1356 else { 1357 /* 1358 * fpdef ['=' test] (',' fpdef ['=' test])* [','] 1359 */ 1360 /* strip trailing comma node */ 1361 if (sym == COMMA) { 1362 res = validate_comma(CHILD(tree, nch-1)); 1363 if (!res) 1364 return 0; 1365 --nch; 1366 } 1367 /* 1368 * fpdef ['=' test] (',' fpdef ['=' test])* 1369 */ 1370 res = validate_fpdef(CHILD(tree, 0)); 1371 ++i; 1372 if (res && (i+2 <= nch) && TYPE(CHILD(tree, i)) == EQUAL) { 1373 res = (validate_equal(CHILD(tree, i)) 1374 && validate_test(CHILD(tree, i+1))); 1375 i += 2; 1376 } 1377 /* 1378 * ... (',' fpdef ['=' test])* 1379 * i ---^^^ 1380 */ 1381 while (res && (nch - i) >= 2) { 1382 res = (validate_comma(CHILD(tree, i)) 1383 && validate_fpdef(CHILD(tree, i+1))); 1384 i += 2; 1385 if (res && (nch - i) >= 2 && TYPE(CHILD(tree, i)) == EQUAL) { 1386 res = (validate_equal(CHILD(tree, i)) 1387 && validate_test(CHILD(tree, i+1))); 1388 i += 2; 1389 } 1390 } 1391 if (res && nch - i != 0) { 1392 res = 0; 1393 err_string("illegal formation for varargslist"); 1394 } 1395 } 1396 } 1397 return res; 1398 } 1399 1400 1401 /* list_iter: list_for | list_if 1402 */ 1403 static int 1404 validate_list_iter(node *tree) 1405 { 1406 int res = (validate_ntype(tree, list_iter) 1407 && validate_numnodes(tree, 1, "list_iter")); 1408 if (res && TYPE(CHILD(tree, 0)) == list_for) 1409 res = validate_list_for(CHILD(tree, 0)); 1410 else 1411 res = validate_list_if(CHILD(tree, 0)); 1412 1413 return res; 1414 } 1415 1416 /* comp_iter: comp_for | comp_if 1417 */ 1418 static int 1419 validate_comp_iter(node *tree) 1420 { 1421 int res = (validate_ntype(tree, comp_iter) 1422 && validate_numnodes(tree, 1, "comp_iter")); 1423 if (res && TYPE(CHILD(tree, 0)) == comp_for) 1424 res = validate_comp_for(CHILD(tree, 0)); 1425 else 1426 res = validate_comp_if(CHILD(tree, 0)); 1427 1428 return res; 1429 } 1430 1431 /* list_for: 'for' exprlist 'in' testlist [list_iter] 1432 */ 1433 static int 1434 validate_list_for(node *tree) 1435 { 1436 int nch = NCH(tree); 1437 int res; 1438 1439 if (nch == 5) 1440 res = validate_list_iter(CHILD(tree, 4)); 1441 else 1442 res = validate_numnodes(tree, 4, "list_for"); 1443 1444 if (res) 1445 res = (validate_name(CHILD(tree, 0), "for") 1446 && validate_exprlist(CHILD(tree, 1)) 1447 && validate_name(CHILD(tree, 2), "in") 1448 && validate_testlist_safe(CHILD(tree, 3))); 1449 1450 return res; 1451 } 1452 1453 /* comp_for: 'for' exprlist 'in' test [comp_iter] 1454 */ 1455 static int 1456 validate_comp_for(node *tree) 1457 { 1458 int nch = NCH(tree); 1459 int res; 1460 1461 if (nch == 5) 1462 res = validate_comp_iter(CHILD(tree, 4)); 1463 else 1464 res = validate_numnodes(tree, 4, "comp_for"); 1465 1466 if (res) 1467 res = (validate_name(CHILD(tree, 0), "for") 1468 && validate_exprlist(CHILD(tree, 1)) 1469 && validate_name(CHILD(tree, 2), "in") 1470 && validate_or_test(CHILD(tree, 3))); 1471 1472 return res; 1473 } 1474 1475 /* list_if: 'if' old_test [list_iter] 1476 */ 1477 static int 1478 validate_list_if(node *tree) 1479 { 1480 int nch = NCH(tree); 1481 int res; 1482 1483 if (nch == 3) 1484 res = validate_list_iter(CHILD(tree, 2)); 1485 else 1486 res = validate_numnodes(tree, 2, "list_if"); 1487 1488 if (res) 1489 res = (validate_name(CHILD(tree, 0), "if") 1490 && validate_old_test(CHILD(tree, 1))); 1491 1492 return res; 1493 } 1494 1495 /* comp_if: 'if' old_test [comp_iter] 1496 */ 1497 static int 1498 validate_comp_if(node *tree) 1499 { 1500 int nch = NCH(tree); 1501 int res; 1502 1503 if (nch == 3) 1504 res = validate_comp_iter(CHILD(tree, 2)); 1505 else 1506 res = validate_numnodes(tree, 2, "comp_if"); 1507 1508 if (res) 1509 res = (validate_name(CHILD(tree, 0), "if") 1510 && validate_old_test(CHILD(tree, 1))); 1511 1512 return res; 1513 } 1514 1515 /* validate_fpdef() 1516 * 1517 * fpdef: 1518 * NAME 1519 * | '(' fplist ')' 1520 */ 1521 static int 1522 validate_fpdef(node *tree) 1523 { 1524 int nch = NCH(tree); 1525 int res = validate_ntype(tree, fpdef); 1526 1527 if (res) { 1528 if (nch == 1) 1529 res = validate_ntype(CHILD(tree, 0), NAME); 1530 else if (nch == 3) 1531 res = (validate_lparen(CHILD(tree, 0)) 1532 && validate_fplist(CHILD(tree, 1)) 1533 && validate_rparen(CHILD(tree, 2))); 1534 else 1535 res = validate_numnodes(tree, 1, "fpdef"); 1536 } 1537 return (res); 1538 } 1539 1540 1541 static int 1542 validate_fplist(node *tree) 1543 { 1544 return (validate_repeating_list(tree, fplist, 1545 validate_fpdef, "fplist")); 1546 } 1547 1548 1549 /* simple_stmt | compound_stmt 1550 * 1551 */ 1552 static int 1553 validate_stmt(node *tree) 1554 { 1555 int res = (validate_ntype(tree, stmt) 1556 && validate_numnodes(tree, 1, "stmt")); 1557 1558 if (res) { 1559 tree = CHILD(tree, 0); 1560 1561 if (TYPE(tree) == simple_stmt) 1562 res = validate_simple_stmt(tree); 1563 else 1564 res = validate_compound_stmt(tree); 1565 } 1566 return (res); 1567 } 1568 1569 1570 /* small_stmt (';' small_stmt)* [';'] NEWLINE 1571 * 1572 */ 1573 static int 1574 validate_simple_stmt(node *tree) 1575 { 1576 int nch = NCH(tree); 1577 int res = (validate_ntype(tree, simple_stmt) 1578 && (nch >= 2) 1579 && validate_small_stmt(CHILD(tree, 0)) 1580 && validate_newline(CHILD(tree, nch - 1))); 1581 1582 if (nch < 2) 1583 res = validate_numnodes(tree, 2, "simple_stmt"); 1584 --nch; /* forget the NEWLINE */ 1585 if (res && is_even(nch)) 1586 res = validate_semi(CHILD(tree, --nch)); 1587 if (res && (nch > 2)) { 1588 int i; 1589 1590 for (i = 1; res && (i < nch); i += 2) 1591 res = (validate_semi(CHILD(tree, i)) 1592 && validate_small_stmt(CHILD(tree, i + 1))); 1593 } 1594 return (res); 1595 } 1596 1597 1598 static int 1599 validate_small_stmt(node *tree) 1600 { 1601 int nch = NCH(tree); 1602 int res = validate_numnodes(tree, 1, "small_stmt"); 1603 1604 if (res) { 1605 int ntype = TYPE(CHILD(tree, 0)); 1606 1607 if ( (ntype == expr_stmt) 1608 || (ntype == print_stmt) 1609 || (ntype == del_stmt) 1610 || (ntype == pass_stmt) 1611 || (ntype == flow_stmt) 1612 || (ntype == import_stmt) 1613 || (ntype == global_stmt) 1614 || (ntype == assert_stmt) 1615 || (ntype == exec_stmt)) 1616 res = validate_node(CHILD(tree, 0)); 1617 else { 1618 res = 0; 1619 err_string("illegal small_stmt child type"); 1620 } 1621 } 1622 else if (nch == 1) { 1623 res = 0; 1624 PyErr_Format(parser_error, 1625 "Unrecognized child node of small_stmt: %d.", 1626 TYPE(CHILD(tree, 0))); 1627 } 1628 return (res); 1629 } 1630 1631 1632 /* compound_stmt: 1633 * if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated 1634 */ 1635 static int 1636 validate_compound_stmt(node *tree) 1637 { 1638 int res = (validate_ntype(tree, compound_stmt) 1639 && validate_numnodes(tree, 1, "compound_stmt")); 1640 int ntype; 1641 1642 if (!res) 1643 return (0); 1644 1645 tree = CHILD(tree, 0); 1646 ntype = TYPE(tree); 1647 if ( (ntype == if_stmt) 1648 || (ntype == while_stmt) 1649 || (ntype == for_stmt) 1650 || (ntype == try_stmt) 1651 || (ntype == with_stmt) 1652 || (ntype == funcdef) 1653 || (ntype == classdef) 1654 || (ntype == decorated)) 1655 res = validate_node(tree); 1656 else { 1657 res = 0; 1658 PyErr_Format(parser_error, 1659 "Illegal compound statement type: %d.", TYPE(tree)); 1660 } 1661 return (res); 1662 } 1663 1664 static int 1665 validate_yield_or_testlist(node *tree) 1666 { 1667 if (TYPE(tree) == yield_expr) 1668 return validate_yield_expr(tree); 1669 else 1670 return validate_testlist(tree); 1671 } 1672 1673 static int 1674 validate_expr_stmt(node *tree) 1675 { 1676 int j; 1677 int nch = NCH(tree); 1678 int res = (validate_ntype(tree, expr_stmt) 1679 && is_odd(nch) 1680 && validate_testlist(CHILD(tree, 0))); 1681 1682 if (res && nch == 3 1683 && TYPE(CHILD(tree, 1)) == augassign) { 1684 res = validate_numnodes(CHILD(tree, 1), 1, "augassign") 1685 && validate_yield_or_testlist(CHILD(tree, 2)); 1686 1687 if (res) { 1688 char *s = STR(CHILD(CHILD(tree, 1), 0)); 1689 1690 res = (strcmp(s, "+=") == 0 1691 || strcmp(s, "-=") == 0 1692 || strcmp(s, "*=") == 0 1693 || strcmp(s, "/=") == 0 1694 || strcmp(s, "//=") == 0 1695 || strcmp(s, "%=") == 0 1696 || strcmp(s, "&=") == 0 1697 || strcmp(s, "|=") == 0 1698 || strcmp(s, "^=") == 0 1699 || strcmp(s, "<<=") == 0 1700 || strcmp(s, ">>=") == 0 1701 || strcmp(s, "**=") == 0); 1702 if (!res) 1703 err_string("illegal augmented assignment operator"); 1704 } 1705 } 1706 else { 1707 for (j = 1; res && (j < nch); j += 2) 1708 res = validate_equal(CHILD(tree, j)) 1709 && validate_yield_or_testlist(CHILD(tree, j + 1)); 1710 } 1711 return (res); 1712 } 1713 1714 1715 /* print_stmt: 1716 * 1717 * 'print' ( [ test (',' test)* [','] ] 1718 * | '>>' test [ (',' test)+ [','] ] ) 1719 */ 1720 static int 1721 validate_print_stmt(node *tree) 1722 { 1723 int nch = NCH(tree); 1724 int res = (validate_ntype(tree, print_stmt) 1725 && (nch > 0) 1726 && validate_name(CHILD(tree, 0), "print")); 1727 1728 if (res && nch > 1) { 1729 int sym = TYPE(CHILD(tree, 1)); 1730 int i = 1; 1731 int allow_trailing_comma = 1; 1732 1733 if (sym == test) 1734 res = validate_test(CHILD(tree, i++)); 1735 else { 1736 if (nch < 3) 1737 res = validate_numnodes(tree, 3, "print_stmt"); 1738 else { 1739 res = (validate_ntype(CHILD(tree, i), RIGHTSHIFT) 1740 && validate_test(CHILD(tree, i+1))); 1741 i += 2; 1742 allow_trailing_comma = 0; 1743 } 1744 } 1745 if (res) { 1746 /* ... (',' test)* [','] */ 1747 while (res && i+2 <= nch) { 1748 res = (validate_comma(CHILD(tree, i)) 1749 && validate_test(CHILD(tree, i+1))); 1750 allow_trailing_comma = 1; 1751 i += 2; 1752 } 1753 if (res && !allow_trailing_comma) 1754 res = validate_numnodes(tree, i, "print_stmt"); 1755 else if (res && i < nch) 1756 res = validate_comma(CHILD(tree, i)); 1757 } 1758 } 1759 return (res); 1760 } 1761 1762 1763 static int 1764 validate_del_stmt(node *tree) 1765 { 1766 return (validate_numnodes(tree, 2, "del_stmt") 1767 && validate_name(CHILD(tree, 0), "del") 1768 && validate_exprlist(CHILD(tree, 1))); 1769 } 1770 1771 1772 static int 1773 validate_return_stmt(node *tree) 1774 { 1775 int nch = NCH(tree); 1776 int res = (validate_ntype(tree, return_stmt) 1777 && ((nch == 1) || (nch == 2)) 1778 && validate_name(CHILD(tree, 0), "return")); 1779 1780 if (res && (nch == 2)) 1781 res = validate_testlist(CHILD(tree, 1)); 1782 1783 return (res); 1784 } 1785 1786 1787 static int 1788 validate_raise_stmt(node *tree) 1789 { 1790 int nch = NCH(tree); 1791 int res = (validate_ntype(tree, raise_stmt) 1792 && ((nch == 1) || (nch == 2) || (nch == 4) || (nch == 6))); 1793 1794 if (res) { 1795 res = validate_name(CHILD(tree, 0), "raise"); 1796 if (res && (nch >= 2)) 1797 res = validate_test(CHILD(tree, 1)); 1798 if (res && nch > 2) { 1799 res = (validate_comma(CHILD(tree, 2)) 1800 && validate_test(CHILD(tree, 3))); 1801 if (res && (nch > 4)) 1802 res = (validate_comma(CHILD(tree, 4)) 1803 && validate_test(CHILD(tree, 5))); 1804 } 1805 } 1806 else 1807 (void) validate_numnodes(tree, 2, "raise"); 1808 if (res && (nch == 4)) 1809 res = (validate_comma(CHILD(tree, 2)) 1810 && validate_test(CHILD(tree, 3))); 1811 1812 return (res); 1813 } 1814 1815 1816 /* yield_expr: 'yield' [testlist] 1817 */ 1818 static int 1819 validate_yield_expr(node *tree) 1820 { 1821 int nch = NCH(tree); 1822 int res = (validate_ntype(tree, yield_expr) 1823 && ((nch == 1) || (nch == 2)) 1824 && validate_name(CHILD(tree, 0), "yield")); 1825 1826 if (res && (nch == 2)) 1827 res = validate_testlist(CHILD(tree, 1)); 1828 1829 return (res); 1830 } 1831 1832 1833 /* yield_stmt: yield_expr 1834 */ 1835 static int 1836 validate_yield_stmt(node *tree) 1837 { 1838 return (validate_ntype(tree, yield_stmt) 1839 && validate_numnodes(tree, 1, "yield_stmt") 1840 && validate_yield_expr(CHILD(tree, 0))); 1841 } 1842 1843 1844 static int 1845 validate_import_as_name(node *tree) 1846 { 1847 int nch = NCH(tree); 1848 int ok = validate_ntype(tree, import_as_name); 1849 1850 if (ok) { 1851 if (nch == 1) 1852 ok = validate_name(CHILD(tree, 0), NULL); 1853 else if (nch == 3) 1854 ok = (validate_name(CHILD(tree, 0), NULL) 1855 && validate_name(CHILD(tree, 1), "as") 1856 && validate_name(CHILD(tree, 2), NULL)); 1857 else 1858 ok = validate_numnodes(tree, 3, "import_as_name"); 1859 } 1860 return ok; 1861 } 1862 1863 1864 /* dotted_name: NAME ("." NAME)* 1865 */ 1866 static int 1867 validate_dotted_name(node *tree) 1868 { 1869 int nch = NCH(tree); 1870 int res = (validate_ntype(tree, dotted_name) 1871 && is_odd(nch) 1872 && validate_name(CHILD(tree, 0), NULL)); 1873 int i; 1874 1875 for (i = 1; res && (i < nch); i += 2) { 1876 res = (validate_dot(CHILD(tree, i)) 1877 && validate_name(CHILD(tree, i+1), NULL)); 1878 } 1879 return res; 1880 } 1881 1882 1883 /* dotted_as_name: dotted_name [NAME NAME] 1884 */ 1885 static int 1886 validate_dotted_as_name(node *tree) 1887 { 1888 int nch = NCH(tree); 1889 int res = validate_ntype(tree, dotted_as_name); 1890 1891 if (res) { 1892 if (nch == 1) 1893 res = validate_dotted_name(CHILD(tree, 0)); 1894 else if (nch == 3) 1895 res = (validate_dotted_name(CHILD(tree, 0)) 1896 && validate_name(CHILD(tree, 1), "as") 1897 && validate_name(CHILD(tree, 2), NULL)); 1898 else { 1899 res = 0; 1900 err_string("illegal number of children for dotted_as_name"); 1901 } 1902 } 1903 return res; 1904 } 1905 1906 1907 /* dotted_as_name (',' dotted_as_name)* */ 1908 static int 1909 validate_dotted_as_names(node *tree) 1910 { 1911 int nch = NCH(tree); 1912 int res = is_odd(nch) && validate_dotted_as_name(CHILD(tree, 0)); 1913 int i; 1914 1915 for (i = 1; res && (i < nch); i += 2) 1916 res = (validate_comma(CHILD(tree, i)) 1917 && validate_dotted_as_name(CHILD(tree, i + 1))); 1918 return (res); 1919 } 1920 1921 1922 /* import_as_name (',' import_as_name)* [','] */ 1923 static int 1924 validate_import_as_names(node *tree) 1925 { 1926 int nch = NCH(tree); 1927 int res = validate_import_as_name(CHILD(tree, 0)); 1928 int i; 1929 1930 for (i = 1; res && (i + 1 < nch); i += 2) 1931 res = (validate_comma(CHILD(tree, i)) 1932 && validate_import_as_name(CHILD(tree, i + 1))); 1933 return (res); 1934 } 1935 1936 1937 /* 'import' dotted_as_names */ 1938 static int 1939 validate_import_name(node *tree) 1940 { 1941 return (validate_ntype(tree, import_name) 1942 && validate_numnodes(tree, 2, "import_name") 1943 && validate_name(CHILD(tree, 0), "import") 1944 && validate_dotted_as_names(CHILD(tree, 1))); 1945 } 1946 1947 /* Helper function to count the number of leading dots in 1948 * 'from ...module import name' 1949 */ 1950 static int 1951 count_from_dots(node *tree) 1952 { 1953 int i; 1954 for (i = 1; i < NCH(tree); i++) 1955 if (TYPE(CHILD(tree, i)) != DOT) 1956 break; 1957 return i-1; 1958 } 1959 1960 /* import_from: ('from' ('.'* dotted_name | '.'+) 1961 * 'import' ('*' | '(' import_as_names ')' | import_as_names)) 1962 */ 1963 static int 1964 validate_import_from(node *tree) 1965 { 1966 int nch = NCH(tree); 1967 int ndots = count_from_dots(tree); 1968 int havename = (TYPE(CHILD(tree, ndots + 1)) == dotted_name); 1969 int offset = ndots + havename; 1970 int res = validate_ntype(tree, import_from) 1971 && (offset >= 1) 1972 && (nch >= 3 + offset) 1973 && validate_name(CHILD(tree, 0), "from") 1974 && (!havename || validate_dotted_name(CHILD(tree, ndots + 1))) 1975 && validate_name(CHILD(tree, offset + 1), "import"); 1976 1977 if (res && TYPE(CHILD(tree, offset + 2)) == LPAR) 1978 res = ((nch == offset + 5) 1979 && validate_lparen(CHILD(tree, offset + 2)) 1980 && validate_import_as_names(CHILD(tree, offset + 3)) 1981 && validate_rparen(CHILD(tree, offset + 4))); 1982 else if (res && TYPE(CHILD(tree, offset + 2)) != STAR) 1983 res = validate_import_as_names(CHILD(tree, offset + 2)); 1984 return (res); 1985 } 1986 1987 1988 /* import_stmt: import_name | import_from */ 1989 static int 1990 validate_import_stmt(node *tree) 1991 { 1992 int nch = NCH(tree); 1993 int res = validate_numnodes(tree, 1, "import_stmt"); 1994 1995 if (res) { 1996 int ntype = TYPE(CHILD(tree, 0)); 1997 1998 if (ntype == import_name || ntype == import_from) 1999 res = validate_node(CHILD(tree, 0)); 2000 else { 2001 res = 0; 2002 err_string("illegal import_stmt child type"); 2003 } 2004 } 2005 else if (nch == 1) { 2006 res = 0; 2007 PyErr_Format(parser_error, 2008 "Unrecognized child node of import_stmt: %d.", 2009 TYPE(CHILD(tree, 0))); 2010 } 2011 return (res); 2012 } 2013 2014 2015 2016 2017 static int 2018 validate_global_stmt(node *tree) 2019 { 2020 int j; 2021 int nch = NCH(tree); 2022 int res = (validate_ntype(tree, global_stmt) 2023 && is_even(nch) && (nch >= 2)); 2024 2025 if (!res && !PyErr_Occurred()) 2026 err_string("illegal global statement"); 2027 2028 if (res) 2029 res = (validate_name(CHILD(tree, 0), "global") 2030 && validate_ntype(CHILD(tree, 1), NAME)); 2031 for (j = 2; res && (j < nch); j += 2) 2032 res = (validate_comma(CHILD(tree, j)) 2033 && validate_ntype(CHILD(tree, j + 1), NAME)); 2034 2035 return (res); 2036 } 2037 2038 2039 /* exec_stmt: 2040 * 2041 * 'exec' expr ['in' test [',' test]] 2042 */ 2043 static int 2044 validate_exec_stmt(node *tree) 2045 { 2046 int nch = NCH(tree); 2047 int res = (validate_ntype(tree, exec_stmt) 2048 && ((nch == 2) || (nch == 4) || (nch == 6)) 2049 && validate_name(CHILD(tree, 0), "exec") 2050 && validate_expr(CHILD(tree, 1))); 2051 2052 if (!res && !PyErr_Occurred()) 2053 err_string("illegal exec statement"); 2054 if (res && (nch > 2)) 2055 res = (validate_name(CHILD(tree, 2), "in") 2056 && validate_test(CHILD(tree, 3))); 2057 if (res && (nch == 6)) 2058 res = (validate_comma(CHILD(tree, 4)) 2059 && validate_test(CHILD(tree, 5))); 2060 2061 return (res); 2062 } 2063 2064 2065 /* assert_stmt: 2066 * 2067 * 'assert' test [',' test] 2068 */ 2069 static int 2070 validate_assert_stmt(node *tree) 2071 { 2072 int nch = NCH(tree); 2073 int res = (validate_ntype(tree, assert_stmt) 2074 && ((nch == 2) || (nch == 4)) 2075 && (validate_name(CHILD(tree, 0), "assert")) 2076 && validate_test(CHILD(tree, 1))); 2077 2078 if (!res && !PyErr_Occurred()) 2079 err_string("illegal assert statement"); 2080 if (res && (nch > 2)) 2081 res = (validate_comma(CHILD(tree, 2)) 2082 && validate_test(CHILD(tree, 3))); 2083 2084 return (res); 2085 } 2086 2087 2088 static int 2089 validate_while(node *tree) 2090 { 2091 int nch = NCH(tree); 2092 int res = (validate_ntype(tree, while_stmt) 2093 && ((nch == 4) || (nch == 7)) 2094 && validate_name(CHILD(tree, 0), "while") 2095 && validate_test(CHILD(tree, 1)) 2096 && validate_colon(CHILD(tree, 2)) 2097 && validate_suite(CHILD(tree, 3))); 2098 2099 if (res && (nch == 7)) 2100 res = (validate_name(CHILD(tree, 4), "else") 2101 && validate_colon(CHILD(tree, 5)) 2102 && validate_suite(CHILD(tree, 6))); 2103 2104 return (res); 2105 } 2106 2107 2108 static int 2109 validate_for(node *tree) 2110 { 2111 int nch = NCH(tree); 2112 int res = (validate_ntype(tree, for_stmt) 2113 && ((nch == 6) || (nch == 9)) 2114 && validate_name(CHILD(tree, 0), "for") 2115 && validate_exprlist(CHILD(tree, 1)) 2116 && validate_name(CHILD(tree, 2), "in") 2117 && validate_testlist(CHILD(tree, 3)) 2118 && validate_colon(CHILD(tree, 4)) 2119 && validate_suite(CHILD(tree, 5))); 2120 2121 if (res && (nch == 9)) 2122 res = (validate_name(CHILD(tree, 6), "else") 2123 && validate_colon(CHILD(tree, 7)) 2124 && validate_suite(CHILD(tree, 8))); 2125 2126 return (res); 2127 } 2128 2129 2130 /* try_stmt: 2131 * 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] 2132 ['finally' ':' suite] 2133 * | 'try' ':' suite 'finally' ':' suite 2134 * 2135 */ 2136 static int 2137 validate_try(node *tree) 2138 { 2139 int nch = NCH(tree); 2140 int pos = 3; 2141 int res = (validate_ntype(tree, try_stmt) 2142 && (nch >= 6) && ((nch % 3) == 0)); 2143 2144 if (res) 2145 res = (validate_name(CHILD(tree, 0), "try") 2146 && validate_colon(CHILD(tree, 1)) 2147 && validate_suite(CHILD(tree, 2)) 2148 && validate_colon(CHILD(tree, nch - 2)) 2149 && validate_suite(CHILD(tree, nch - 1))); 2150 else if (!PyErr_Occurred()) { 2151 const char* name = "except"; 2152 if (TYPE(CHILD(tree, nch - 3)) != except_clause) 2153 name = STR(CHILD(tree, nch - 3)); 2154 2155 PyErr_Format(parser_error, 2156 "Illegal number of children for try/%s node.", name); 2157 } 2158 /* Handle try/finally statement */ 2159 if (res && (TYPE(CHILD(tree, pos)) == NAME) && 2160 (strcmp(STR(CHILD(tree, pos)), "finally") == 0)) { 2161 res = (validate_numnodes(tree, 6, "try/finally") 2162 && validate_colon(CHILD(tree, 4)) 2163 && validate_suite(CHILD(tree, 5))); 2164 return (res); 2165 } 2166 /* try/except statement: skip past except_clause sections */ 2167 while (res && pos < nch && (TYPE(CHILD(tree, pos)) == except_clause)) { 2168 res = (validate_except_clause(CHILD(tree, pos)) 2169 && validate_colon(CHILD(tree, pos + 1)) 2170 && validate_suite(CHILD(tree, pos + 2))); 2171 pos += 3; 2172 } 2173 /* skip else clause */ 2174 if (res && pos < nch && (TYPE(CHILD(tree, pos)) == NAME) && 2175 (strcmp(STR(CHILD(tree, pos)), "else") == 0)) { 2176 res = (validate_colon(CHILD(tree, pos + 1)) 2177 && validate_suite(CHILD(tree, pos + 2))); 2178 pos += 3; 2179 } 2180 if (res && pos < nch) { 2181 /* last clause must be a finally */ 2182 res = (validate_name(CHILD(tree, pos), "finally") 2183 && validate_numnodes(tree, pos + 3, "try/except/finally") 2184 && validate_colon(CHILD(tree, pos + 1)) 2185 && validate_suite(CHILD(tree, pos + 2))); 2186 } 2187 return (res); 2188 } 2189 2190 2191 static int 2192 validate_except_clause(node *tree) 2193 { 2194 int nch = NCH(tree); 2195 int res = (validate_ntype(tree, except_clause) 2196 && ((nch == 1) || (nch == 2) || (nch == 4)) 2197 && validate_name(CHILD(tree, 0), "except")); 2198 2199 if (res && (nch > 1)) 2200 res = validate_test(CHILD(tree, 1)); 2201 if (res && (nch == 4)) { 2202 if (TYPE(CHILD(tree, 2)) == NAME) 2203 res = validate_name(CHILD(tree, 2), "as"); 2204 else 2205 res = validate_comma(CHILD(tree, 2)); 2206 res = res && validate_test(CHILD(tree, 3)); 2207 } 2208 return (res); 2209 } 2210 2211 2212 static int 2213 validate_test(node *tree) 2214 { 2215 int nch = NCH(tree); 2216 int res = validate_ntype(tree, test) && is_odd(nch); 2217 2218 if (res && (TYPE(CHILD(tree, 0)) == lambdef)) 2219 res = ((nch == 1) 2220 && validate_lambdef(CHILD(tree, 0))); 2221 else if (res) { 2222 res = validate_or_test(CHILD(tree, 0)); 2223 res = (res && (nch == 1 || (nch == 5 && 2224 validate_name(CHILD(tree, 1), "if") && 2225 validate_or_test(CHILD(tree, 2)) && 2226 validate_name(CHILD(tree, 3), "else") && 2227 validate_test(CHILD(tree, 4))))); 2228 } 2229 return (res); 2230 } 2231 2232 static int 2233 validate_old_test(node *tree) 2234 { 2235 int nch = NCH(tree); 2236 int res = validate_ntype(tree, old_test) && (nch == 1); 2237 2238 if (res && (TYPE(CHILD(tree, 0)) == old_lambdef)) 2239 res = (validate_old_lambdef(CHILD(tree, 0))); 2240 else if (res) { 2241 res = (validate_or_test(CHILD(tree, 0))); 2242 } 2243 return (res); 2244 } 2245 2246 static int 2247 validate_or_test(node *tree) 2248 { 2249 int nch = NCH(tree); 2250 int res = validate_ntype(tree, or_test) && is_odd(nch); 2251 2252 if (res) { 2253 int pos; 2254 res = validate_and_test(CHILD(tree, 0)); 2255 for (pos = 1; res && (pos < nch); pos += 2) 2256 res = (validate_name(CHILD(tree, pos), "or") 2257 && validate_and_test(CHILD(tree, pos + 1))); 2258 } 2259 return (res); 2260 } 2261 2262 2263 static int 2264 validate_and_test(node *tree) 2265 { 2266 int pos; 2267 int nch = NCH(tree); 2268 int res = (validate_ntype(tree, and_test) 2269 && is_odd(nch) 2270 && validate_not_test(CHILD(tree, 0))); 2271 2272 for (pos = 1; res && (pos < nch); pos += 2) 2273 res = (validate_name(CHILD(tree, pos), "and") 2274 && validate_not_test(CHILD(tree, 0))); 2275 2276 return (res); 2277 } 2278 2279 2280 static int 2281 validate_not_test(node *tree) 2282 { 2283 int nch = NCH(tree); 2284 int res = validate_ntype(tree, not_test) && ((nch == 1) || (nch == 2)); 2285 2286 if (res) { 2287 if (nch == 2) 2288 res = (validate_name(CHILD(tree, 0), "not") 2289 && validate_not_test(CHILD(tree, 1))); 2290 else if (nch == 1) 2291 res = validate_comparison(CHILD(tree, 0)); 2292 } 2293 return (res); 2294 } 2295 2296 2297 static int 2298 validate_comparison(node *tree) 2299 { 2300 int pos; 2301 int nch = NCH(tree); 2302 int res = (validate_ntype(tree, comparison) 2303 && is_odd(nch) 2304 && validate_expr(CHILD(tree, 0))); 2305 2306 for (pos = 1; res && (pos < nch); pos += 2) 2307 res = (validate_comp_op(CHILD(tree, pos)) 2308 && validate_expr(CHILD(tree, pos + 1))); 2309 2310 return (res); 2311 } 2312 2313 2314 static int 2315 validate_comp_op(node *tree) 2316 { 2317 int res = 0; 2318 int nch = NCH(tree); 2319 2320 if (!validate_ntype(tree, comp_op)) 2321 return (0); 2322 if (nch == 1) { 2323 /* 2324 * Only child will be a terminal with a well-defined symbolic name 2325 * or a NAME with a string of either 'is' or 'in' 2326 */ 2327 tree = CHILD(tree, 0); 2328 switch (TYPE(tree)) { 2329 case LESS: 2330 case GREATER: 2331 case EQEQUAL: 2332 case EQUAL: 2333 case LESSEQUAL: 2334 case GREATEREQUAL: 2335 case NOTEQUAL: 2336 res = 1; 2337 break; 2338 case NAME: 2339 res = ((strcmp(STR(tree), "in") == 0) 2340 || (strcmp(STR(tree), "is") == 0)); 2341 if (!res) { 2342 PyErr_Format(parser_error, 2343 "illegal operator '%s'", STR(tree)); 2344 } 2345 break; 2346 default: 2347 err_string("illegal comparison operator type"); 2348 break; 2349 } 2350 } 2351 else if ((res = validate_numnodes(tree, 2, "comp_op")) != 0) { 2352 res = (validate_ntype(CHILD(tree, 0), NAME) 2353 && validate_ntype(CHILD(tree, 1), NAME) 2354 && (((strcmp(STR(CHILD(tree, 0)), "is") == 0) 2355 && (strcmp(STR(CHILD(tree, 1)), "not") == 0)) 2356 || ((strcmp(STR(CHILD(tree, 0)), "not") == 0) 2357 && (strcmp(STR(CHILD(tree, 1)), "in") == 0)))); 2358 if (!res && !PyErr_Occurred()) 2359 err_string("unknown comparison operator"); 2360 } 2361 return (res); 2362 } 2363 2364 2365 static int 2366 validate_expr(node *tree) 2367 { 2368 int j; 2369 int nch = NCH(tree); 2370 int res = (validate_ntype(tree, expr) 2371 && is_odd(nch) 2372 && validate_xor_expr(CHILD(tree, 0))); 2373 2374 for (j = 2; res && (j < nch); j += 2) 2375 res = (validate_xor_expr(CHILD(tree, j)) 2376 && validate_vbar(CHILD(tree, j - 1))); 2377 2378 return (res); 2379 } 2380 2381 2382 static int 2383 validate_xor_expr(node *tree) 2384 { 2385 int j; 2386 int nch = NCH(tree); 2387 int res = (validate_ntype(tree, xor_expr) 2388 && is_odd(nch) 2389 && validate_and_expr(CHILD(tree, 0))); 2390 2391 for (j = 2; res && (j < nch); j += 2) 2392 res = (validate_circumflex(CHILD(tree, j - 1)) 2393 && validate_and_expr(CHILD(tree, j))); 2394 2395 return (res); 2396 } 2397 2398 2399 static int 2400 validate_and_expr(node *tree) 2401 { 2402 int pos; 2403 int nch = NCH(tree); 2404 int res = (validate_ntype(tree, and_expr) 2405 && is_odd(nch) 2406 && validate_shift_expr(CHILD(tree, 0))); 2407 2408 for (pos = 1; res && (pos < nch); pos += 2) 2409 res = (validate_ampersand(CHILD(tree, pos)) 2410 && validate_shift_expr(CHILD(tree, pos + 1))); 2411 2412 return (res); 2413 } 2414 2415 2416 static int 2417 validate_chain_two_ops(node *tree, int (*termvalid)(node *), int op1, int op2) 2418 { 2419 int pos = 1; 2420 int nch = NCH(tree); 2421 int res = (is_odd(nch) 2422 && (*termvalid)(CHILD(tree, 0))); 2423 2424 for ( ; res && (pos < nch); pos += 2) { 2425 if (TYPE(CHILD(tree, pos)) != op1) 2426 res = validate_ntype(CHILD(tree, pos), op2); 2427 if (res) 2428 res = (*termvalid)(CHILD(tree, pos + 1)); 2429 } 2430 return (res); 2431 } 2432 2433 2434 static int 2435 validate_shift_expr(node *tree) 2436 { 2437 return (validate_ntype(tree, shift_expr) 2438 && validate_chain_two_ops(tree, validate_arith_expr, 2439 LEFTSHIFT, RIGHTSHIFT)); 2440 } 2441 2442 2443 static int 2444 validate_arith_expr(node *tree) 2445 { 2446 return (validate_ntype(tree, arith_expr) 2447 && validate_chain_two_ops(tree, validate_term, PLUS, MINUS)); 2448 } 2449 2450 2451 static int 2452 validate_term(node *tree) 2453 { 2454 int pos = 1; 2455 int nch = NCH(tree); 2456 int res = (validate_ntype(tree, term) 2457 && is_odd(nch) 2458 && validate_factor(CHILD(tree, 0))); 2459 2460 for ( ; res && (pos < nch); pos += 2) 2461 res = (((TYPE(CHILD(tree, pos)) == STAR) 2462 || (TYPE(CHILD(tree, pos)) == SLASH) 2463 || (TYPE(CHILD(tree, pos)) == DOUBLESLASH) 2464 || (TYPE(CHILD(tree, pos)) == PERCENT)) 2465 && validate_factor(CHILD(tree, pos + 1))); 2466 2467 return (res); 2468 } 2469 2470 2471 /* factor: 2472 * 2473 * factor: ('+'|'-'|'~') factor | power 2474 */ 2475 static int 2476 validate_factor(node *tree) 2477 { 2478 int nch = NCH(tree); 2479 int res = (validate_ntype(tree, factor) 2480 && (((nch == 2) 2481 && ((TYPE(CHILD(tree, 0)) == PLUS) 2482 || (TYPE(CHILD(tree, 0)) == MINUS) 2483 || (TYPE(CHILD(tree, 0)) == TILDE)) 2484 && validate_factor(CHILD(tree, 1))) 2485 || ((nch == 1) 2486 && validate_power(CHILD(tree, 0))))); 2487 return (res); 2488 } 2489 2490 2491 /* power: 2492 * 2493 * power: atom trailer* ('**' factor)* 2494 */ 2495 static int 2496 validate_power(node *tree) 2497 { 2498 int pos = 1; 2499 int nch = NCH(tree); 2500 int res = (validate_ntype(tree, power) && (nch >= 1) 2501 && validate_atom(CHILD(tree, 0))); 2502 2503 while (res && (pos < nch) && (TYPE(CHILD(tree, pos)) == trailer)) 2504 res = validate_trailer(CHILD(tree, pos++)); 2505 if (res && (pos < nch)) { 2506 if (!is_even(nch - pos)) { 2507 err_string("illegal number of nodes for 'power'"); 2508 return (0); 2509 } 2510 for ( ; res && (pos < (nch - 1)); pos += 2) 2511 res = (validate_doublestar(CHILD(tree, pos)) 2512 && validate_factor(CHILD(tree, pos + 1))); 2513 } 2514 return (res); 2515 } 2516 2517 2518 static int 2519 validate_atom(node *tree) 2520 { 2521 int pos; 2522 int nch = NCH(tree); 2523 int res = validate_ntype(tree, atom); 2524 2525 if (res && nch < 1) 2526 res = validate_numnodes(tree, nch+1, "atom"); 2527 if (res) { 2528 switch (TYPE(CHILD(tree, 0))) { 2529 case LPAR: 2530 res = ((nch <= 3) 2531 && (validate_rparen(CHILD(tree, nch - 1)))); 2532 2533 if (res && (nch == 3)) { 2534 if (TYPE(CHILD(tree, 1))==yield_expr) 2535 res = validate_yield_expr(CHILD(tree, 1)); 2536 else 2537 res = validate_testlist_comp(CHILD(tree, 1)); 2538 } 2539 break; 2540 case LSQB: 2541 if (nch == 2) 2542 res = validate_ntype(CHILD(tree, 1), RSQB); 2543 else if (nch == 3) 2544 res = (validate_listmaker(CHILD(tree, 1)) 2545 && validate_ntype(CHILD(tree, 2), RSQB)); 2546 else { 2547 res = 0; 2548 err_string("illegal list display atom"); 2549 } 2550 break; 2551 case LBRACE: 2552 res = ((nch <= 3) 2553 && validate_ntype(CHILD(tree, nch - 1), RBRACE)); 2554 2555 if (res && (nch == 3)) 2556 res = validate_dictorsetmaker(CHILD(tree, 1)); 2557 break; 2558 case BACKQUOTE: 2559 res = ((nch == 3) 2560 && validate_testlist1(CHILD(tree, 1)) 2561 && validate_ntype(CHILD(tree, 2), BACKQUOTE)); 2562 break; 2563 case NAME: 2564 case NUMBER: 2565 res = (nch == 1); 2566 break; 2567 case STRING: 2568 for (pos = 1; res && (pos < nch); ++pos) 2569 res = validate_ntype(CHILD(tree, pos), STRING); 2570 break; 2571 default: 2572 res = 0; 2573 break; 2574 } 2575 } 2576 return (res); 2577 } 2578 2579 2580 /* listmaker: 2581 * test ( list_for | (',' test)* [','] ) 2582 */ 2583 static int 2584 validate_listmaker(node *tree) 2585 { 2586 int nch = NCH(tree); 2587 int ok = nch; 2588 2589 if (nch == 0) 2590 err_string("missing child nodes of listmaker"); 2591 else 2592 ok = validate_test(CHILD(tree, 0)); 2593 2594 /* 2595 * list_for | (',' test)* [','] 2596 */ 2597 if (nch == 2 && TYPE(CHILD(tree, 1)) == list_for) 2598 ok = validate_list_for(CHILD(tree, 1)); 2599 else { 2600 /* (',' test)* [','] */ 2601 int i = 1; 2602 while (ok && nch - i >= 2) { 2603 ok = (validate_comma(CHILD(tree, i)) 2604 && validate_test(CHILD(tree, i+1))); 2605 i += 2; 2606 } 2607 if (ok && i == nch-1) 2608 ok = validate_comma(CHILD(tree, i)); 2609 else if (i != nch) { 2610 ok = 0; 2611 err_string("illegal trailing nodes for listmaker"); 2612 } 2613 } 2614 return ok; 2615 } 2616 2617 /* testlist_comp: 2618 * test ( comp_for | (',' test)* [','] ) 2619 */ 2620 static int 2621 validate_testlist_comp(node *tree) 2622 { 2623 int nch = NCH(tree); 2624 int ok = nch; 2625 2626 if (nch == 0) 2627 err_string("missing child nodes of testlist_comp"); 2628 else { 2629 ok = validate_test(CHILD(tree, 0)); 2630 } 2631 2632 /* 2633 * comp_for | (',' test)* [','] 2634 */ 2635 if (nch == 2 && TYPE(CHILD(tree, 1)) == comp_for) 2636 ok = validate_comp_for(CHILD(tree, 1)); 2637 else { 2638 /* (',' test)* [','] */ 2639 int i = 1; 2640 while (ok && nch - i >= 2) { 2641 ok = (validate_comma(CHILD(tree, i)) 2642 && validate_test(CHILD(tree, i+1))); 2643 i += 2; 2644 } 2645 if (ok && i == nch-1) 2646 ok = validate_comma(CHILD(tree, i)); 2647 else if (i != nch) { 2648 ok = 0; 2649 err_string("illegal trailing nodes for testlist_comp"); 2650 } 2651 } 2652 return ok; 2653 } 2654 2655 /* decorator: 2656 * '@' dotted_name [ '(' [arglist] ')' ] NEWLINE 2657 */ 2658 static int 2659 validate_decorator(node *tree) 2660 { 2661 int ok; 2662 int nch = NCH(tree); 2663 ok = (validate_ntype(tree, decorator) && 2664 (nch == 3 || nch == 5 || nch == 6) && 2665 validate_at(CHILD(tree, 0)) && 2666 validate_dotted_name(CHILD(tree, 1)) && 2667 validate_newline(RCHILD(tree, -1))); 2668 2669 if (ok && nch != 3) { 2670 ok = (validate_lparen(CHILD(tree, 2)) && 2671 validate_rparen(RCHILD(tree, -2))); 2672 2673 if (ok && nch == 6) 2674 ok = validate_arglist(CHILD(tree, 3)); 2675 } 2676 2677 return ok; 2678 } 2679 2680 /* decorators: 2681 * decorator+ 2682 */ 2683 static int 2684 validate_decorators(node *tree) 2685 { 2686 int i, nch, ok; 2687 nch = NCH(tree); 2688 ok = validate_ntype(tree, decorators) && nch >= 1; 2689 2690 for (i = 0; ok && i < nch; ++i) 2691 ok = validate_decorator(CHILD(tree, i)); 2692 2693 return ok; 2694 } 2695 2696 /* with_item: 2697 * test ['as' expr] 2698 */ 2699 static int 2700 validate_with_item(node *tree) 2701 { 2702 int nch = NCH(tree); 2703 int ok = (validate_ntype(tree, with_item) 2704 && (nch == 1 || nch == 3) 2705 && validate_test(CHILD(tree, 0))); 2706 if (ok && nch == 3) 2707 ok = (validate_name(CHILD(tree, 1), "as") 2708 && validate_expr(CHILD(tree, 2))); 2709 return ok; 2710 } 2711 2712 /* with_stmt: 2713 * 0 1 ... -2 -1 2714 * 'with' with_item (',' with_item)* ':' suite 2715 */ 2716 static int 2717 validate_with_stmt(node *tree) 2718 { 2719 int i; 2720 int nch = NCH(tree); 2721 int ok = (validate_ntype(tree, with_stmt) 2722 && (nch % 2 == 0) 2723 && validate_name(CHILD(tree, 0), "with") 2724 && validate_colon(RCHILD(tree, -2)) 2725 && validate_suite(RCHILD(tree, -1))); 2726 for (i = 1; ok && i < nch - 2; i += 2) 2727 ok = validate_with_item(CHILD(tree, i)); 2728 return ok; 2729 } 2730 2731 /* funcdef: 2732 * 2733 * -5 -4 -3 -2 -1 2734 * 'def' NAME parameters ':' suite 2735 */ 2736 static int 2737 validate_funcdef(node *tree) 2738 { 2739 int nch = NCH(tree); 2740 int ok = (validate_ntype(tree, funcdef) 2741 && (nch == 5) 2742 && validate_name(RCHILD(tree, -5), "def") 2743 && validate_ntype(RCHILD(tree, -4), NAME) 2744 && validate_colon(RCHILD(tree, -2)) 2745 && validate_parameters(RCHILD(tree, -3)) 2746 && validate_suite(RCHILD(tree, -1))); 2747 return ok; 2748 } 2749 2750 2751 /* decorated 2752 * decorators (classdef | funcdef) 2753 */ 2754 static int 2755 validate_decorated(node *tree) 2756 { 2757 int nch = NCH(tree); 2758 int ok = (validate_ntype(tree, decorated) 2759 && (nch == 2) 2760 && validate_decorators(RCHILD(tree, -2))); 2761 if (TYPE(RCHILD(tree, -1)) == funcdef) 2762 ok = ok && validate_funcdef(RCHILD(tree, -1)); 2763 else 2764 ok = ok && validate_class(RCHILD(tree, -1)); 2765 return ok; 2766 } 2767 2768 static int 2769 validate_lambdef(node *tree) 2770 { 2771 int nch = NCH(tree); 2772 int res = (validate_ntype(tree, lambdef) 2773 && ((nch == 3) || (nch == 4)) 2774 && validate_name(CHILD(tree, 0), "lambda") 2775 && validate_colon(CHILD(tree, nch - 2)) 2776 && validate_test(CHILD(tree, nch - 1))); 2777 2778 if (res && (nch == 4)) 2779 res = validate_varargslist(CHILD(tree, 1)); 2780 else if (!res && !PyErr_Occurred()) 2781 (void) validate_numnodes(tree, 3, "lambdef"); 2782 2783 return (res); 2784 } 2785 2786 2787 static int 2788 validate_old_lambdef(node *tree) 2789 { 2790 int nch = NCH(tree); 2791 int res = (validate_ntype(tree, old_lambdef) 2792 && ((nch == 3) || (nch == 4)) 2793 && validate_name(CHILD(tree, 0), "lambda") 2794 && validate_colon(CHILD(tree, nch - 2)) 2795 && validate_test(CHILD(tree, nch - 1))); 2796 2797 if (res && (nch == 4)) 2798 res = validate_varargslist(CHILD(tree, 1)); 2799 else if (!res && !PyErr_Occurred()) 2800 (void) validate_numnodes(tree, 3, "old_lambdef"); 2801 2802 return (res); 2803 } 2804 2805 2806 /* arglist: 2807 * 2808 * (argument ',')* (argument [','] | '*' test [',' '**' test] | '**' test) 2809 */ 2810 static int 2811 validate_arglist(node *tree) 2812 { 2813 int nch = NCH(tree); 2814 int i = 0; 2815 int ok = 1; 2816 2817 if (nch <= 0) 2818 /* raise the right error from having an invalid number of children */ 2819 return validate_numnodes(tree, nch + 1, "arglist"); 2820 2821 if (nch > 1) { 2822 for (i=0; i<nch; i++) { 2823 if (TYPE(CHILD(tree, i)) == argument) { 2824 node *ch = CHILD(tree, i); 2825 if (NCH(ch) == 2 && TYPE(CHILD(ch, 1)) == comp_for) { 2826 err_string("need '(', ')' for generator expression"); 2827 return 0; 2828 } 2829 } 2830 } 2831 } 2832 2833 while (ok && nch-i >= 2) { 2834 /* skip leading (argument ',') */ 2835 ok = (validate_argument(CHILD(tree, i)) 2836 && validate_comma(CHILD(tree, i+1))); 2837 if (ok) 2838 i += 2; 2839 else 2840 PyErr_Clear(); 2841 } 2842 ok = 1; 2843 if (nch-i > 0) { 2844 /* 2845 * argument | '*' test [',' '**' test] | '**' test 2846 */ 2847 int sym = TYPE(CHILD(tree, i)); 2848 2849 if (sym == argument) { 2850 ok = validate_argument(CHILD(tree, i)); 2851 if (ok && i+1 != nch) { 2852 err_string("illegal arglist specification" 2853 " (extra stuff on end)"); 2854 ok = 0; 2855 } 2856 } 2857 else if (sym == STAR) { 2858 ok = validate_star(CHILD(tree, i)); 2859 if (ok && (nch-i == 2)) 2860 ok = validate_test(CHILD(tree, i+1)); 2861 else if (ok && (nch-i == 5)) 2862 ok = (validate_test(CHILD(tree, i+1)) 2863 && validate_comma(CHILD(tree, i+2)) 2864 && validate_doublestar(CHILD(tree, i+3)) 2865 && validate_test(CHILD(tree, i+4))); 2866 else { 2867 err_string("illegal use of '*' in arglist"); 2868 ok = 0; 2869 } 2870 } 2871 else if (sym == DOUBLESTAR) { 2872 if (nch-i == 2) 2873 ok = (validate_doublestar(CHILD(tree, i)) 2874 && validate_test(CHILD(tree, i+1))); 2875 else { 2876 err_string("illegal use of '**' in arglist"); 2877 ok = 0; 2878 } 2879 } 2880 else { 2881 err_string("illegal arglist specification"); 2882 ok = 0; 2883 } 2884 } 2885 return (ok); 2886 } 2887 2888 2889 2890 /* argument: 2891 * 2892 * [test '='] test [comp_for] 2893 */ 2894 static int 2895 validate_argument(node *tree) 2896 { 2897 int nch = NCH(tree); 2898 int res = (validate_ntype(tree, argument) 2899 && ((nch == 1) || (nch == 2) || (nch == 3)) 2900 && validate_test(CHILD(tree, 0))); 2901 2902 if (res && (nch == 2)) 2903 res = validate_comp_for(CHILD(tree, 1)); 2904 else if (res && (nch == 3)) 2905 res = (validate_equal(CHILD(tree, 1)) 2906 && validate_test(CHILD(tree, 2))); 2907 2908 return (res); 2909 } 2910 2911 2912 2913 /* trailer: 2914 * 2915 * '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME 2916 */ 2917 static int 2918 validate_trailer(node *tree) 2919 { 2920 int nch = NCH(tree); 2921 int res = validate_ntype(tree, trailer) && ((nch == 2) || (nch == 3)); 2922 2923 if (res) { 2924 switch (TYPE(CHILD(tree, 0))) { 2925 case LPAR: 2926 res = validate_rparen(CHILD(tree, nch - 1)); 2927 if (res && (nch == 3)) 2928 res = validate_arglist(CHILD(tree, 1)); 2929 break; 2930 case LSQB: 2931 res = (validate_numnodes(tree, 3, "trailer") 2932 && validate_subscriptlist(CHILD(tree, 1)) 2933 && validate_ntype(CHILD(tree, 2), RSQB)); 2934 break; 2935 case DOT: 2936 res = (validate_numnodes(tree, 2, "trailer") 2937 && validate_ntype(CHILD(tree, 1), NAME)); 2938 break; 2939 default: 2940 res = 0; 2941 break; 2942 } 2943 } 2944 else { 2945 (void) validate_numnodes(tree, 2, "trailer"); 2946 } 2947 return (res); 2948 } 2949 2950 2951 /* subscriptlist: 2952 * 2953 * subscript (',' subscript)* [','] 2954 */ 2955 static int 2956 validate_subscriptlist(node *tree) 2957 { 2958 return (validate_repeating_list(tree, subscriptlist, 2959 validate_subscript, "subscriptlist")); 2960 } 2961 2962 2963 /* subscript: 2964 * 2965 * '.' '.' '.' | test | [test] ':' [test] [sliceop] 2966 */ 2967 static int 2968 validate_subscript(node *tree) 2969 { 2970 int offset = 0; 2971 int nch = NCH(tree); 2972 int res = validate_ntype(tree, subscript) && (nch >= 1) && (nch <= 4); 2973 2974 if (!res) { 2975 if (!PyErr_Occurred()) 2976 err_string("invalid number of arguments for subscript node"); 2977 return (0); 2978 } 2979 if (TYPE(CHILD(tree, 0)) == DOT) 2980 /* take care of ('.' '.' '.') possibility */ 2981 return (validate_numnodes(tree, 3, "subscript") 2982 && validate_dot(CHILD(tree, 0)) 2983 && validate_dot(CHILD(tree, 1)) 2984 && validate_dot(CHILD(tree, 2))); 2985 if (nch == 1) { 2986 if (TYPE(CHILD(tree, 0)) == test) 2987 res = validate_test(CHILD(tree, 0)); 2988 else 2989 res = validate_colon(CHILD(tree, 0)); 2990 return (res); 2991 } 2992 /* Must be [test] ':' [test] [sliceop], 2993 * but at least one of the optional components will 2994 * be present, but we don't know which yet. 2995 */ 2996 if ((TYPE(CHILD(tree, 0)) != COLON) || (nch == 4)) { 2997 res = validate_test(CHILD(tree, 0)); 2998 offset = 1; 2999 } 3000 if (res) 3001 res = validate_colon(CHILD(tree, offset)); 3002 if (res) { 3003 int rem = nch - ++offset; 3004 if (rem) { 3005 if (TYPE(CHILD(tree, offset)) == test) { 3006 res = validate_test(CHILD(tree, offset)); 3007 ++offset; 3008 --rem; 3009 } 3010 if (res && rem) 3011 res = validate_sliceop(CHILD(tree, offset)); 3012 } 3013 } 3014 return (res); 3015 } 3016 3017 3018 static int 3019 validate_sliceop(node *tree) 3020 { 3021 int nch = NCH(tree); 3022 int res = ((nch == 1) || validate_numnodes(tree, 2, "sliceop")) 3023 && validate_ntype(tree, sliceop); 3024 if (!res && !PyErr_Occurred()) { 3025 res = validate_numnodes(tree, 1, "sliceop"); 3026 } 3027 if (res) 3028 res = validate_colon(CHILD(tree, 0)); 3029 if (res && (nch == 2)) 3030 res = validate_test(CHILD(tree, 1)); 3031 3032 return (res); 3033 } 3034 3035 3036 static int 3037 validate_exprlist(node *tree) 3038 { 3039 return (validate_repeating_list(tree, exprlist, 3040 validate_expr, "exprlist")); 3041 } 3042 3043 3044 /* 3045 * dictorsetmaker: 3046 * 3047 * (test ':' test (comp_for | (',' test ':' test)* [','])) | 3048 * (test (comp_for | (',' test)* [','])) 3049 */ 3050 static int 3051 validate_dictorsetmaker(node *tree) 3052 { 3053 int nch = NCH(tree); 3054 int ok = validate_ntype(tree, dictorsetmaker); 3055 int i = 0; 3056 int check_trailing_comma = 0; 3057 3058 assert(nch > 0); 3059 3060 if (ok && (nch == 1 || TYPE(CHILD(tree, 1)) == COMMA)) { 3061 /* We got a set: 3062 * test (',' test)* [','] 3063 */ 3064 ok = validate_test(CHILD(tree, i++)); 3065 while (ok && nch - i >= 2) { 3066 ok = (validate_comma(CHILD(tree, i)) 3067 && validate_test(CHILD(tree, i+1))); 3068 i += 2; 3069 } 3070 check_trailing_comma = 1; 3071 } 3072 else if (ok && TYPE(CHILD(tree, 1)) == comp_for) { 3073 /* We got a set comprehension: 3074 * test comp_for 3075 */ 3076 ok = (validate_test(CHILD(tree, 0)) 3077 && validate_comp_for(CHILD(tree, 1))); 3078 } 3079 else if (ok && NCH(tree) > 3 && TYPE(CHILD(tree, 3)) == comp_for) { 3080 /* We got a dict comprehension: 3081 * test ':' test comp_for 3082 */ 3083 ok = (validate_test(CHILD(tree, 0)) 3084 && validate_colon(CHILD(tree, 1)) 3085 && validate_test(CHILD(tree, 2)) 3086 && validate_comp_for(CHILD(tree, 3))); 3087 } 3088 else if (ok) { 3089 /* We got a dict: 3090 * test ':' test (',' test ':' test)* [','] 3091 */ 3092 if (nch >= 3) { 3093 ok = (validate_test(CHILD(tree, i)) 3094 && validate_colon(CHILD(tree, i+1)) 3095 && validate_test(CHILD(tree, i+2))); 3096 i += 3; 3097 } 3098 else { 3099 ok = 0; 3100 err_string("illegal number of nodes for dictorsetmaker"); 3101 } 3102 3103 while (ok && nch - i >= 4) { 3104 ok = (validate_comma(CHILD(tree, i)) 3105 && validate_test(CHILD(tree, i+1)) 3106 && validate_colon(CHILD(tree, i+2)) 3107 && validate_test(CHILD(tree, i+3))); 3108 i += 4; 3109 } 3110 check_trailing_comma = 1; 3111 } 3112 if (ok && check_trailing_comma) { 3113 if (i == nch-1) 3114 ok = validate_comma(CHILD(tree, i)); 3115 else if (i != nch) { 3116 ok = 0; 3117 err_string("illegal trailing nodes for dictorsetmaker"); 3118 } 3119 } 3120 3121 return ok; 3122 } 3123 3124 3125 static int 3126 validate_eval_input(node *tree) 3127 { 3128 int pos; 3129 int nch = NCH(tree); 3130 int res = (validate_ntype(tree, eval_input) 3131 && (nch >= 2) 3132 && validate_testlist(CHILD(tree, 0)) 3133 && validate_ntype(CHILD(tree, nch - 1), ENDMARKER)); 3134 3135 for (pos = 1; res && (pos < (nch - 1)); ++pos) 3136 res = validate_ntype(CHILD(tree, pos), NEWLINE); 3137 3138 return (res); 3139 } 3140 3141 3142 static int 3143 validate_node(node *tree) 3144 { 3145 int nch = 0; /* num. children on current node */ 3146 int res = 1; /* result value */ 3147 node* next = 0; /* node to process after this one */ 3148 3149 while (res && (tree != 0)) { 3150 nch = NCH(tree); 3151 next = 0; 3152 switch (TYPE(tree)) { 3153 /* 3154 * Definition nodes. 3155 */ 3156 case funcdef: 3157 res = validate_funcdef(tree); 3158 break; 3159 case with_stmt: 3160 res = validate_with_stmt(tree); 3161 break; 3162 case classdef: 3163 res = validate_class(tree); 3164 break; 3165 case decorated: 3166 res = validate_decorated(tree); 3167 break; 3168 /* 3169 * "Trivial" parse tree nodes. 3170 * (Why did I call these trivial?) 3171 */ 3172 case stmt: 3173 res = validate_stmt(tree); 3174 break; 3175 case small_stmt: 3176 /* 3177 * expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt 3178 * | import_stmt | global_stmt | exec_stmt | assert_stmt 3179 */ 3180 res = validate_small_stmt(tree); 3181 break; 3182 case flow_stmt: 3183 res = (validate_numnodes(tree, 1, "flow_stmt") 3184 && ((TYPE(CHILD(tree, 0)) == break_stmt) 3185 || (TYPE(CHILD(tree, 0)) == continue_stmt) 3186 || (TYPE(CHILD(tree, 0)) == yield_stmt) 3187 || (TYPE(CHILD(tree, 0)) == return_stmt) 3188 || (TYPE(CHILD(tree, 0)) == raise_stmt))); 3189 if (res) 3190 next = CHILD(tree, 0); 3191 else if (nch == 1) 3192 err_string("illegal flow_stmt type"); 3193 break; 3194 case yield_stmt: 3195 res = validate_yield_stmt(tree); 3196 break; 3197 /* 3198 * Compound statements. 3199 */ 3200 case simple_stmt: 3201 res = validate_simple_stmt(tree); 3202 break; 3203 case compound_stmt: 3204 res = validate_compound_stmt(tree); 3205 break; 3206 /* 3207 * Fundamental statements. 3208 */ 3209 case expr_stmt: 3210 res = validate_expr_stmt(tree); 3211 break; 3212 case print_stmt: 3213 res = validate_print_stmt(tree); 3214 break; 3215 case del_stmt: 3216 res = validate_del_stmt(tree); 3217 break; 3218 case pass_stmt: 3219 res = (validate_numnodes(tree, 1, "pass") 3220 && validate_name(CHILD(tree, 0), "pass")); 3221 break; 3222 case break_stmt: 3223 res = (validate_numnodes(tree, 1, "break") 3224 && validate_name(CHILD(tree, 0), "break")); 3225 break; 3226 case continue_stmt: 3227 res = (validate_numnodes(tree, 1, "continue") 3228 && validate_name(CHILD(tree, 0), "continue")); 3229 break; 3230 case return_stmt: 3231 res = validate_return_stmt(tree); 3232 break; 3233 case raise_stmt: 3234 res = validate_raise_stmt(tree); 3235 break; 3236 case import_stmt: 3237 res = validate_import_stmt(tree); 3238 break; 3239 case import_name: 3240 res = validate_import_name(tree); 3241 break; 3242 case import_from: 3243 res = validate_import_from(tree); 3244 break; 3245 case global_stmt: 3246 res = validate_global_stmt(tree); 3247 break; 3248 case exec_stmt: 3249 res = validate_exec_stmt(tree); 3250 break; 3251 case assert_stmt: 3252 res = validate_assert_stmt(tree); 3253 break; 3254 case if_stmt: 3255 res = validate_if(tree); 3256 break; 3257 case while_stmt: 3258 res = validate_while(tree); 3259 break; 3260 case for_stmt: 3261 res = validate_for(tree); 3262 break; 3263 case try_stmt: 3264 res = validate_try(tree); 3265 break; 3266 case suite: 3267 res = validate_suite(tree); 3268 break; 3269 /* 3270 * Expression nodes. 3271 */ 3272 case testlist: 3273 res = validate_testlist(tree); 3274 break; 3275 case yield_expr: 3276 res = validate_yield_expr(tree); 3277 break; 3278 case testlist1: 3279 res = validate_testlist1(tree); 3280 break; 3281 case test: 3282 res = validate_test(tree); 3283 break; 3284 case and_test: 3285 res = validate_and_test(tree); 3286 break; 3287 case not_test: 3288 res = validate_not_test(tree); 3289 break; 3290 case comparison: 3291 res = validate_comparison(tree); 3292 break; 3293 case exprlist: 3294 res = validate_exprlist(tree); 3295 break; 3296 case comp_op: 3297 res = validate_comp_op(tree); 3298 break; 3299 case expr: 3300 res = validate_expr(tree); 3301 break; 3302 case xor_expr: 3303 res = validate_xor_expr(tree); 3304 break; 3305 case and_expr: 3306 res = validate_and_expr(tree); 3307 break; 3308 case shift_expr: 3309 res = validate_shift_expr(tree); 3310 break; 3311 case arith_expr: 3312 res = validate_arith_expr(tree); 3313 break; 3314 case term: 3315 res = validate_term(tree); 3316 break; 3317 case factor: 3318 res = validate_factor(tree); 3319 break; 3320 case power: 3321 res = validate_power(tree); 3322 break; 3323 case atom: 3324 res = validate_atom(tree); 3325 break; 3326 3327 default: 3328 /* Hopefully never reached! */ 3329 err_string("unrecognized node type"); 3330 res = 0; 3331 break; 3332 } 3333 tree = next; 3334 } 3335 return (res); 3336 } 3337 3338 3339 static int 3340 validate_expr_tree(node *tree) 3341 { 3342 int res = validate_eval_input(tree); 3343 3344 if (!res && !PyErr_Occurred()) 3345 err_string("could not validate expression tuple"); 3346 3347 return (res); 3348 } 3349 3350 3351 /* file_input: 3352 * (NEWLINE | stmt)* ENDMARKER 3353 */ 3354 static int 3355 validate_file_input(node *tree) 3356 { 3357 int j; 3358 int nch = NCH(tree) - 1; 3359 int res = ((nch >= 0) 3360 && validate_ntype(CHILD(tree, nch), ENDMARKER)); 3361 3362 for (j = 0; res && (j < nch); ++j) { 3363 if (TYPE(CHILD(tree, j)) == stmt) 3364 res = validate_stmt(CHILD(tree, j)); 3365 else 3366 res = validate_newline(CHILD(tree, j)); 3367 } 3368 /* This stays in to prevent any internal failures from getting to the 3369 * user. Hopefully, this won't be needed. If a user reports getting 3370 * this, we have some debugging to do. 3371 */ 3372 if (!res && !PyErr_Occurred()) 3373 err_string("VALIDATION FAILURE: report this to the maintainer!"); 3374 3375 return (res); 3376 } 3377 3378 static int 3379 validate_encoding_decl(node *tree) 3380 { 3381 int nch = NCH(tree); 3382 int res = ((nch == 1) 3383 && validate_file_input(CHILD(tree, 0))); 3384 3385 if (!res && !PyErr_Occurred()) 3386 err_string("Error Parsing encoding_decl"); 3387 3388 return res; 3389 } 3390 3391 static PyObject* 3392 pickle_constructor = NULL; 3393 3394 3395 static PyObject* 3396 parser__pickler(PyObject *self, PyObject *args) 3397 { 3398 NOTE(ARGUNUSED(self)) 3399 PyObject *result = NULL; 3400 PyObject *st = NULL; 3401 PyObject *empty_dict = NULL; 3402 3403 if (PyArg_ParseTuple(args, "O!:_pickler", &PyST_Type, &st)) { 3404 PyObject *newargs; 3405 PyObject *tuple; 3406 3407 if ((empty_dict = PyDict_New()) == NULL) 3408 goto finally; 3409 if ((newargs = Py_BuildValue("Oi", st, 1)) == NULL) 3410 goto finally; 3411 tuple = parser_st2tuple((PyST_Object*)NULL, newargs, empty_dict); 3412 if (tuple != NULL) { 3413 result = Py_BuildValue("O(O)", pickle_constructor, tuple); 3414 Py_DECREF(tuple); 3415 } 3416 Py_DECREF(newargs); 3417 } 3418 finally: 3419 Py_XDECREF(empty_dict); 3420 3421 return (result); 3422 } 3423 3424 3425 /* Functions exported by this module. Most of this should probably 3426 * be converted into an ST object with methods, but that is better 3427 * done directly in Python, allowing subclasses to be created directly. 3428 * We'd really have to write a wrapper around it all anyway to allow 3429 * inheritance. 3430 */ 3431 static PyMethodDef parser_functions[] = { 3432 {"ast2tuple", (PyCFunction)parser_ast2tuple, PUBLIC_METHOD_TYPE, 3433 PyDoc_STR("Creates a tuple-tree representation of an ST.")}, 3434 {"ast2list", (PyCFunction)parser_ast2list, PUBLIC_METHOD_TYPE, 3435 PyDoc_STR("Creates a list-tree representation of an ST.")}, 3436 {"compileast", (PyCFunction)parser_compileast,PUBLIC_METHOD_TYPE, 3437 PyDoc_STR("Compiles an ST object into a code object.")}, 3438 {"compilest", (PyCFunction)parser_compilest, PUBLIC_METHOD_TYPE, 3439 PyDoc_STR("Compiles an ST object into a code object.")}, 3440 {"expr", (PyCFunction)parser_expr, PUBLIC_METHOD_TYPE, 3441 PyDoc_STR("Creates an ST object from an expression.")}, 3442 {"isexpr", (PyCFunction)parser_isexpr, PUBLIC_METHOD_TYPE, 3443 PyDoc_STR("Determines if an ST object was created from an expression.")}, 3444 {"issuite", (PyCFunction)parser_issuite, PUBLIC_METHOD_TYPE, 3445 PyDoc_STR("Determines if an ST object was created from a suite.")}, 3446 {"suite", (PyCFunction)parser_suite, PUBLIC_METHOD_TYPE, 3447 PyDoc_STR("Creates an ST object from a suite.")}, 3448 {"sequence2ast", (PyCFunction)parser_tuple2ast, PUBLIC_METHOD_TYPE, 3449 PyDoc_STR("Creates an ST object from a tree representation.")}, 3450 {"sequence2st", (PyCFunction)parser_tuple2st, PUBLIC_METHOD_TYPE, 3451 PyDoc_STR("Creates an ST object from a tree representation.")}, 3452 {"st2tuple", (PyCFunction)parser_st2tuple, PUBLIC_METHOD_TYPE, 3453 PyDoc_STR("Creates a tuple-tree representation of an ST.")}, 3454 {"st2list", (PyCFunction)parser_st2list, PUBLIC_METHOD_TYPE, 3455 PyDoc_STR("Creates a list-tree representation of an ST.")}, 3456 {"tuple2ast", (PyCFunction)parser_tuple2ast, PUBLIC_METHOD_TYPE, 3457 PyDoc_STR("Creates an ST object from a tree representation.")}, 3458 {"tuple2st", (PyCFunction)parser_tuple2st, PUBLIC_METHOD_TYPE, 3459 PyDoc_STR("Creates an ST object from a tree representation.")}, 3460 3461 /* private stuff: support pickle module */ 3462 {"_pickler", (PyCFunction)parser__pickler, METH_VARARGS, 3463 PyDoc_STR("Returns the pickle magic to allow ST objects to be pickled.")}, 3464 3465 {NULL, NULL, 0, NULL} 3466 }; 3467 3468 3469 PyMODINIT_FUNC initparser(void); /* supply a prototype */ 3470 3471 PyMODINIT_FUNC 3472 initparser(void) 3473 { 3474 PyObject *module, *copyreg; 3475 3476 Py_TYPE(&PyST_Type) = &PyType_Type; 3477 module = Py_InitModule("parser", parser_functions); 3478 if (module == NULL) 3479 return; 3480 3481 if (parser_error == 0) 3482 parser_error = PyErr_NewException("parser.ParserError", NULL, NULL); 3483 3484 if (parser_error == 0) 3485 /* caller will check PyErr_Occurred() */ 3486 return; 3487 /* CAUTION: The code next used to skip bumping the refcount on 3488 * parser_error. That's a disaster if initparser() gets called more 3489 * than once. By incref'ing, we ensure that each module dict that 3490 * gets created owns its reference to the shared parser_error object, 3491 * and the file static parser_error vrbl owns a reference too. 3492 */ 3493 Py_INCREF(parser_error); 3494 if (PyModule_AddObject(module, "ParserError", parser_error) != 0) 3495 return; 3496 3497 Py_INCREF(&PyST_Type); 3498 PyModule_AddObject(module, "ASTType", (PyObject*)&PyST_Type); 3499 Py_INCREF(&PyST_Type); 3500 PyModule_AddObject(module, "STType", (PyObject*)&PyST_Type); 3501 3502 PyModule_AddStringConstant(module, "__copyright__", 3503 parser_copyright_string); 3504 PyModule_AddStringConstant(module, "__doc__", 3505 parser_doc_string); 3506 PyModule_AddStringConstant(module, "__version__", 3507 parser_version_string); 3508 3509 /* Register to support pickling. 3510 * If this fails, the import of this module will fail because an 3511 * exception will be raised here; should we clear the exception? 3512 */ 3513 copyreg = PyImport_ImportModuleNoBlock("copy_reg"); 3514 if (copyreg != NULL) { 3515 PyObject *func, *pickler; 3516 3517 func = PyObject_GetAttrString(copyreg, "pickle"); 3518 pickle_constructor = PyObject_GetAttrString(module, "sequence2st"); 3519 pickler = PyObject_GetAttrString(module, "_pickler"); 3520 Py_XINCREF(pickle_constructor); 3521 if ((func != NULL) && (pickle_constructor != NULL) 3522 && (pickler != NULL)) { 3523 PyObject *res; 3524 3525 res = PyObject_CallFunctionObjArgs(func, &PyST_Type, pickler, 3526 pickle_constructor, NULL); 3527 Py_XDECREF(res); 3528 } 3529 Py_XDECREF(func); 3530 Py_XDECREF(pickle_constructor); 3531 Py_XDECREF(pickler); 3532 Py_DECREF(copyreg); 3533 } 3534 } 3535
