1 #! /usr/bin/env python 2 """Generate C code from an ASDL description.""" 3 4 # TO DO 5 # handle fields that have a type but no name 6 7 import os, sys 8 9 import asdl 10 11 TABSIZE = 8 12 MAX_COL = 80 13 14 def get_c_type(name): 15 """Return a string for the C name of the type. 16 17 This function special cases the default types provided by asdl: 18 identifier, string, int, bool. 19 """ 20 # XXX ack! need to figure out where Id is useful and where string 21 if isinstance(name, asdl.Id): 22 name = name.value 23 if name in asdl.builtin_types: 24 return name 25 else: 26 return "%s_ty" % name 27 28 def reflow_lines(s, depth): 29 """Reflow the line s indented depth tabs. 30 31 Return a sequence of lines where no line extends beyond MAX_COL 32 when properly indented. The first line is properly indented based 33 exclusively on depth * TABSIZE. All following lines -- these are 34 the reflowed lines generated by this function -- start at the same 35 column as the first character beyond the opening { in the first 36 line. 37 """ 38 size = MAX_COL - depth * TABSIZE 39 if len(s) < size: 40 return [s] 41 42 lines = [] 43 cur = s 44 padding = "" 45 while len(cur) > size: 46 i = cur.rfind(' ', 0, size) 47 # XXX this should be fixed for real 48 if i == -1 and 'GeneratorExp' in cur: 49 i = size + 3 50 assert i != -1, "Impossible line %d to reflow: %r" % (size, s) 51 lines.append(padding + cur[:i]) 52 if len(lines) == 1: 53 # find new size based on brace 54 j = cur.find('{', 0, i) 55 if j >= 0: 56 j += 2 # account for the brace and the space after it 57 size -= j 58 padding = " " * j 59 else: 60 j = cur.find('(', 0, i) 61 if j >= 0: 62 j += 1 # account for the paren (no space after it) 63 size -= j 64 padding = " " * j 65 cur = cur[i+1:] 66 else: 67 lines.append(padding + cur) 68 return lines 69 70 def is_simple(sum): 71 """Return True if a sum is a simple. 72 73 A sum is simple if its types have no fields, e.g. 74 unaryop = Invert | Not | UAdd | USub 75 """ 76 for t in sum.types: 77 if t.fields: 78 return False 79 return True 80 81 82 class EmitVisitor(asdl.VisitorBase): 83 """Visit that emits lines""" 84 85 def __init__(self, file): 86 self.file = file 87 super(EmitVisitor, self).__init__() 88 89 def emit(self, s, depth, reflow=True): 90 # XXX reflow long lines? 91 if reflow: 92 lines = reflow_lines(s, depth) 93 else: 94 lines = [s] 95 for line in lines: 96 line = (" " * TABSIZE * depth) + line + "\n" 97 self.file.write(line) 98 99 100 class TypeDefVisitor(EmitVisitor): 101 def visitModule(self, mod): 102 for dfn in mod.dfns: 103 self.visit(dfn) 104 105 def visitType(self, type, depth=0): 106 self.visit(type.value, type.name, depth) 107 108 def visitSum(self, sum, name, depth): 109 if is_simple(sum): 110 self.simple_sum(sum, name, depth) 111 else: 112 self.sum_with_constructors(sum, name, depth) 113 114 def simple_sum(self, sum, name, depth): 115 enum = [] 116 for i in range(len(sum.types)): 117 type = sum.types[i] 118 enum.append("%s=%d" % (type.name, i + 1)) 119 enums = ", ".join(enum) 120 ctype = get_c_type(name) 121 s = "typedef enum _%s { %s } %s;" % (name, enums, ctype) 122 self.emit(s, depth) 123 self.emit("", depth) 124 125 def sum_with_constructors(self, sum, name, depth): 126 ctype = get_c_type(name) 127 s = "typedef struct _%(name)s *%(ctype)s;" % locals() 128 self.emit(s, depth) 129 self.emit("", depth) 130 131 def visitProduct(self, product, name, depth): 132 ctype = get_c_type(name) 133 s = "typedef struct _%(name)s *%(ctype)s;" % locals() 134 self.emit(s, depth) 135 self.emit("", depth) 136 137 138 class StructVisitor(EmitVisitor): 139 """Visitor to generate typedefs for AST.""" 140 141 def visitModule(self, mod): 142 for dfn in mod.dfns: 143 self.visit(dfn) 144 145 def visitType(self, type, depth=0): 146 self.visit(type.value, type.name, depth) 147 148 def visitSum(self, sum, name, depth): 149 if not is_simple(sum): 150 self.sum_with_constructors(sum, name, depth) 151 152 def sum_with_constructors(self, sum, name, depth): 153 def emit(s, depth=depth): 154 self.emit(s % sys._getframe(1).f_locals, depth) 155 enum = [] 156 for i in range(len(sum.types)): 157 type = sum.types[i] 158 enum.append("%s_kind=%d" % (type.name, i + 1)) 159 160 emit("enum _%(name)s_kind {" + ", ".join(enum) + "};") 161 162 emit("struct _%(name)s {") 163 emit("enum _%(name)s_kind kind;", depth + 1) 164 emit("union {", depth + 1) 165 for t in sum.types: 166 self.visit(t, depth + 2) 167 emit("} v;", depth + 1) 168 for field in sum.attributes: 169 # rudimentary attribute handling 170 type = str(field.type) 171 assert type in asdl.builtin_types, type 172 emit("%s %s;" % (type, field.name), depth + 1); 173 emit("};") 174 emit("") 175 176 def visitConstructor(self, cons, depth): 177 if cons.fields: 178 self.emit("struct {", depth) 179 for f in cons.fields: 180 self.visit(f, depth + 1) 181 self.emit("} %s;" % cons.name, depth) 182 self.emit("", depth) 183 else: 184 # XXX not sure what I want here, nothing is probably fine 185 pass 186 187 def visitField(self, field, depth): 188 # XXX need to lookup field.type, because it might be something 189 # like a builtin... 190 ctype = get_c_type(field.type) 191 name = field.name 192 if field.seq: 193 if field.type.value in ('cmpop',): 194 self.emit("asdl_int_seq *%(name)s;" % locals(), depth) 195 else: 196 self.emit("asdl_seq *%(name)s;" % locals(), depth) 197 else: 198 self.emit("%(ctype)s %(name)s;" % locals(), depth) 199 200 def visitProduct(self, product, name, depth): 201 self.emit("struct _%(name)s {" % locals(), depth) 202 for f in product.fields: 203 self.visit(f, depth + 1) 204 self.emit("};", depth) 205 self.emit("", depth) 206 207 208 class PrototypeVisitor(EmitVisitor): 209 """Generate function prototypes for the .h file""" 210 211 def visitModule(self, mod): 212 for dfn in mod.dfns: 213 self.visit(dfn) 214 215 def visitType(self, type): 216 self.visit(type.value, type.name) 217 218 def visitSum(self, sum, name): 219 if is_simple(sum): 220 pass # XXX 221 else: 222 for t in sum.types: 223 self.visit(t, name, sum.attributes) 224 225 def get_args(self, fields): 226 """Return list of C argument into, one for each field. 227 228 Argument info is 3-tuple of a C type, variable name, and flag 229 that is true if type can be NULL. 230 """ 231 args = [] 232 unnamed = {} 233 for f in fields: 234 if f.name is None: 235 name = f.type 236 c = unnamed[name] = unnamed.get(name, 0) + 1 237 if c > 1: 238 name = "name%d" % (c - 1) 239 else: 240 name = f.name 241 # XXX should extend get_c_type() to handle this 242 if f.seq: 243 if f.type.value in ('cmpop',): 244 ctype = "asdl_int_seq *" 245 else: 246 ctype = "asdl_seq *" 247 else: 248 ctype = get_c_type(f.type) 249 args.append((ctype, name, f.opt or f.seq)) 250 return args 251 252 def visitConstructor(self, cons, type, attrs): 253 args = self.get_args(cons.fields) 254 attrs = self.get_args(attrs) 255 ctype = get_c_type(type) 256 self.emit_function(cons.name, ctype, args, attrs) 257 258 def emit_function(self, name, ctype, args, attrs, union=True): 259 args = args + attrs 260 if args: 261 argstr = ", ".join(["%s %s" % (atype, aname) 262 for atype, aname, opt in args]) 263 argstr += ", PyArena *arena" 264 else: 265 argstr = "PyArena *arena" 266 margs = "a0" 267 for i in range(1, len(args)+1): 268 margs += ", a%d" % i 269 self.emit("#define %s(%s) _Py_%s(%s)" % (name, margs, name, margs), 0, 270 reflow=False) 271 self.emit("%s _Py_%s(%s);" % (ctype, name, argstr), False) 272 273 def visitProduct(self, prod, name): 274 self.emit_function(name, get_c_type(name), 275 self.get_args(prod.fields), [], union=False) 276 277 278 class FunctionVisitor(PrototypeVisitor): 279 """Visitor to generate constructor functions for AST.""" 280 281 def emit_function(self, name, ctype, args, attrs, union=True): 282 def emit(s, depth=0, reflow=True): 283 self.emit(s, depth, reflow) 284 argstr = ", ".join(["%s %s" % (atype, aname) 285 for atype, aname, opt in args + attrs]) 286 if argstr: 287 argstr += ", PyArena *arena" 288 else: 289 argstr = "PyArena *arena" 290 self.emit("%s" % ctype, 0) 291 emit("%s(%s)" % (name, argstr)) 292 emit("{") 293 emit("%s p;" % ctype, 1) 294 for argtype, argname, opt in args: 295 # XXX hack alert: false is allowed for a bool 296 if not opt and not (argtype == "bool" or argtype == "int"): 297 emit("if (!%s) {" % argname, 1) 298 emit("PyErr_SetString(PyExc_ValueError,", 2) 299 msg = "field %s is required for %s" % (argname, name) 300 emit(' "%s");' % msg, 301 2, reflow=False) 302 emit('return NULL;', 2) 303 emit('}', 1) 304 305 emit("p = (%s)PyArena_Malloc(arena, sizeof(*p));" % ctype, 1); 306 emit("if (!p)", 1) 307 emit("return NULL;", 2) 308 if union: 309 self.emit_body_union(name, args, attrs) 310 else: 311 self.emit_body_struct(name, args, attrs) 312 emit("return p;", 1) 313 emit("}") 314 emit("") 315 316 def emit_body_union(self, name, args, attrs): 317 def emit(s, depth=0, reflow=True): 318 self.emit(s, depth, reflow) 319 emit("p->kind = %s_kind;" % name, 1) 320 for argtype, argname, opt in args: 321 emit("p->v.%s.%s = %s;" % (name, argname, argname), 1) 322 for argtype, argname, opt in attrs: 323 emit("p->%s = %s;" % (argname, argname), 1) 324 325 def emit_body_struct(self, name, args, attrs): 326 def emit(s, depth=0, reflow=True): 327 self.emit(s, depth, reflow) 328 for argtype, argname, opt in args: 329 emit("p->%s = %s;" % (argname, argname), 1) 330 assert not attrs 331 332 333 class PickleVisitor(EmitVisitor): 334 335 def visitModule(self, mod): 336 for dfn in mod.dfns: 337 self.visit(dfn) 338 339 def visitType(self, type): 340 self.visit(type.value, type.name) 341 342 def visitSum(self, sum, name): 343 pass 344 345 def visitProduct(self, sum, name): 346 pass 347 348 def visitConstructor(self, cons, name): 349 pass 350 351 def visitField(self, sum): 352 pass 353 354 355 class Obj2ModPrototypeVisitor(PickleVisitor): 356 def visitProduct(self, prod, name): 357 code = "static int obj2ast_%s(PyObject* obj, %s* out, PyArena* arena);" 358 self.emit(code % (name, get_c_type(name)), 0) 359 360 visitSum = visitProduct 361 362 363 class Obj2ModVisitor(PickleVisitor): 364 def funcHeader(self, name): 365 ctype = get_c_type(name) 366 self.emit("int", 0) 367 self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) 368 self.emit("{", 0) 369 self.emit("PyObject* tmp = NULL;", 1) 370 self.emit("int isinstance;", 1) 371 self.emit("", 0) 372 373 def sumTrailer(self, name): 374 self.emit("", 0) 375 self.emit("tmp = PyObject_Repr(obj);", 1) 376 # there's really nothing more we can do if this fails ... 377 self.emit("if (tmp == NULL) goto failed;", 1) 378 error = "expected some sort of %s, but got %%.400s" % name 379 format = "PyErr_Format(PyExc_TypeError, \"%s\", PyString_AS_STRING(tmp));" 380 self.emit(format % error, 1, reflow=False) 381 self.emit("failed:", 0) 382 self.emit("Py_XDECREF(tmp);", 1) 383 self.emit("return 1;", 1) 384 self.emit("}", 0) 385 self.emit("", 0) 386 387 def simpleSum(self, sum, name): 388 self.funcHeader(name) 389 for t in sum.types: 390 line = ("isinstance = PyObject_IsInstance(obj, " 391 "(PyObject *)%s_type);") 392 self.emit(line % (t.name,), 1) 393 self.emit("if (isinstance == -1) {", 1) 394 self.emit("return 1;", 2) 395 self.emit("}", 1) 396 self.emit("if (isinstance) {", 1) 397 self.emit("*out = %s;" % t.name, 2) 398 self.emit("return 0;", 2) 399 self.emit("}", 1) 400 self.sumTrailer(name) 401 402 def buildArgs(self, fields): 403 return ", ".join(fields + ["arena"]) 404 405 def complexSum(self, sum, name): 406 self.funcHeader(name) 407 for a in sum.attributes: 408 self.visitAttributeDeclaration(a, name, sum=sum) 409 self.emit("", 0) 410 # XXX: should we only do this for 'expr'? 411 self.emit("if (obj == Py_None) {", 1) 412 self.emit("*out = NULL;", 2) 413 self.emit("return 0;", 2) 414 self.emit("}", 1) 415 for a in sum.attributes: 416 self.visitField(a, name, sum=sum, depth=1) 417 for t in sum.types: 418 line = "isinstance = PyObject_IsInstance(obj, (PyObject*)%s_type);" 419 self.emit(line % (t.name,), 1) 420 self.emit("if (isinstance == -1) {", 1) 421 self.emit("return 1;", 2) 422 self.emit("}", 1) 423 self.emit("if (isinstance) {", 1) 424 for f in t.fields: 425 self.visitFieldDeclaration(f, t.name, sum=sum, depth=2) 426 self.emit("", 0) 427 for f in t.fields: 428 self.visitField(f, t.name, sum=sum, depth=2) 429 args = [f.name.value for f in t.fields] + [a.name.value for a in sum.attributes] 430 self.emit("*out = %s(%s);" % (t.name, self.buildArgs(args)), 2) 431 self.emit("if (*out == NULL) goto failed;", 2) 432 self.emit("return 0;", 2) 433 self.emit("}", 1) 434 self.sumTrailer(name) 435 436 def visitAttributeDeclaration(self, a, name, sum=sum): 437 ctype = get_c_type(a.type) 438 self.emit("%s %s;" % (ctype, a.name), 1) 439 440 def visitSum(self, sum, name): 441 if is_simple(sum): 442 self.simpleSum(sum, name) 443 else: 444 self.complexSum(sum, name) 445 446 def visitProduct(self, prod, name): 447 ctype = get_c_type(name) 448 self.emit("int", 0) 449 self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) 450 self.emit("{", 0) 451 self.emit("PyObject* tmp = NULL;", 1) 452 for f in prod.fields: 453 self.visitFieldDeclaration(f, name, prod=prod, depth=1) 454 self.emit("", 0) 455 for f in prod.fields: 456 self.visitField(f, name, prod=prod, depth=1) 457 args = [f.name.value for f in prod.fields] 458 self.emit("*out = %s(%s);" % (name, self.buildArgs(args)), 1) 459 self.emit("return 0;", 1) 460 self.emit("failed:", 0) 461 self.emit("Py_XDECREF(tmp);", 1) 462 self.emit("return 1;", 1) 463 self.emit("}", 0) 464 self.emit("", 0) 465 466 def visitFieldDeclaration(self, field, name, sum=None, prod=None, depth=0): 467 ctype = get_c_type(field.type) 468 if field.seq: 469 if self.isSimpleType(field): 470 self.emit("asdl_int_seq* %s;" % field.name, depth) 471 else: 472 self.emit("asdl_seq* %s;" % field.name, depth) 473 else: 474 ctype = get_c_type(field.type) 475 self.emit("%s %s;" % (ctype, field.name), depth) 476 477 def isSimpleSum(self, field): 478 # XXX can the members of this list be determined automatically? 479 return field.type.value in ('expr_context', 'boolop', 'operator', 480 'unaryop', 'cmpop') 481 482 def isNumeric(self, field): 483 return get_c_type(field.type) in ("int", "bool") 484 485 def isSimpleType(self, field): 486 return self.isSimpleSum(field) or self.isNumeric(field) 487 488 def visitField(self, field, name, sum=None, prod=None, depth=0): 489 ctype = get_c_type(field.type) 490 self.emit("if (PyObject_HasAttrString(obj, \"%s\")) {" % field.name, depth) 491 self.emit("int res;", depth+1) 492 if field.seq: 493 self.emit("Py_ssize_t len;", depth+1) 494 self.emit("Py_ssize_t i;", depth+1) 495 self.emit("tmp = PyObject_GetAttrString(obj, \"%s\");" % field.name, depth+1) 496 self.emit("if (tmp == NULL) goto failed;", depth+1) 497 if field.seq: 498 self.emit("if (!PyList_Check(tmp)) {", depth+1) 499 self.emit("PyErr_Format(PyExc_TypeError, \"%s field \\\"%s\\\" must " 500 "be a list, not a %%.200s\", tmp->ob_type->tp_name);" % 501 (name, field.name), 502 depth+2, reflow=False) 503 self.emit("goto failed;", depth+2) 504 self.emit("}", depth+1) 505 self.emit("len = PyList_GET_SIZE(tmp);", depth+1) 506 if self.isSimpleType(field): 507 self.emit("%s = asdl_int_seq_new(len, arena);" % field.name, depth+1) 508 else: 509 self.emit("%s = asdl_seq_new(len, arena);" % field.name, depth+1) 510 self.emit("if (%s == NULL) goto failed;" % field.name, depth+1) 511 self.emit("for (i = 0; i < len; i++) {", depth+1) 512 self.emit("%s value;" % ctype, depth+2) 513 self.emit("res = obj2ast_%s(PyList_GET_ITEM(tmp, i), &value, arena);" % 514 field.type, depth+2, reflow=False) 515 self.emit("if (res != 0) goto failed;", depth+2) 516 self.emit("if (len != PyList_GET_SIZE(tmp)) {", depth+2) 517 self.emit("PyErr_SetString(PyExc_RuntimeError, \"%s field \\\"%s\\\" " 518 "changed size during iteration\");" % 519 (name, field.name), 520 depth+3, reflow=False) 521 self.emit("goto failed;", depth+3) 522 self.emit("}", depth+2) 523 self.emit("asdl_seq_SET(%s, i, value);" % field.name, depth+2) 524 self.emit("}", depth+1) 525 else: 526 self.emit("res = obj2ast_%s(tmp, &%s, arena);" % 527 (field.type, field.name), depth+1) 528 self.emit("if (res != 0) goto failed;", depth+1) 529 530 self.emit("Py_XDECREF(tmp);", depth+1) 531 self.emit("tmp = NULL;", depth+1) 532 self.emit("} else {", depth) 533 if not field.opt: 534 message = "required field \\\"%s\\\" missing from %s" % (field.name, name) 535 format = "PyErr_SetString(PyExc_TypeError, \"%s\");" 536 self.emit(format % message, depth+1, reflow=False) 537 self.emit("return 1;", depth+1) 538 else: 539 if self.isNumeric(field): 540 self.emit("%s = 0;" % field.name, depth+1) 541 elif not self.isSimpleType(field): 542 self.emit("%s = NULL;" % field.name, depth+1) 543 else: 544 raise TypeError("could not determine the default value for %s" % field.name) 545 self.emit("}", depth) 546 547 548 class MarshalPrototypeVisitor(PickleVisitor): 549 550 def prototype(self, sum, name): 551 ctype = get_c_type(name) 552 self.emit("static int marshal_write_%s(PyObject **, int *, %s);" 553 % (name, ctype), 0) 554 555 visitProduct = visitSum = prototype 556 557 558 class PyTypesDeclareVisitor(PickleVisitor): 559 560 def visitProduct(self, prod, name): 561 self.emit("static PyTypeObject *%s_type;" % name, 0) 562 self.emit("static PyObject* ast2obj_%s(void*);" % name, 0) 563 if prod.fields: 564 self.emit("static char *%s_fields[]={" % name,0) 565 for f in prod.fields: 566 self.emit('"%s",' % f.name, 1) 567 self.emit("};", 0) 568 569 def visitSum(self, sum, name): 570 self.emit("static PyTypeObject *%s_type;" % name, 0) 571 if sum.attributes: 572 self.emit("static char *%s_attributes[] = {" % name, 0) 573 for a in sum.attributes: 574 self.emit('"%s",' % a.name, 1) 575 self.emit("};", 0) 576 ptype = "void*" 577 if is_simple(sum): 578 ptype = get_c_type(name) 579 tnames = [] 580 for t in sum.types: 581 tnames.append(str(t.name)+"_singleton") 582 tnames = ", *".join(tnames) 583 self.emit("static PyObject *%s;" % tnames, 0) 584 self.emit("static PyObject* ast2obj_%s(%s);" % (name, ptype), 0) 585 for t in sum.types: 586 self.visitConstructor(t, name) 587 588 def visitConstructor(self, cons, name): 589 self.emit("static PyTypeObject *%s_type;" % cons.name, 0) 590 if cons.fields: 591 self.emit("static char *%s_fields[]={" % cons.name, 0) 592 for t in cons.fields: 593 self.emit('"%s",' % t.name, 1) 594 self.emit("};",0) 595 596 class PyTypesVisitor(PickleVisitor): 597 598 def visitModule(self, mod): 599 self.emit(""" 600 static int 601 ast_type_init(PyObject *self, PyObject *args, PyObject *kw) 602 { 603 Py_ssize_t i, numfields = 0; 604 int res = -1; 605 PyObject *key, *value, *fields; 606 fields = PyObject_GetAttrString((PyObject*)Py_TYPE(self), "_fields"); 607 if (!fields) 608 PyErr_Clear(); 609 if (fields) { 610 numfields = PySequence_Size(fields); 611 if (numfields == -1) 612 goto cleanup; 613 } 614 res = 0; /* if no error occurs, this stays 0 to the end */ 615 if (PyTuple_GET_SIZE(args) > 0) { 616 if (numfields != PyTuple_GET_SIZE(args)) { 617 PyErr_Format(PyExc_TypeError, "%.400s constructor takes %s" 618 "%zd positional argument%s", 619 Py_TYPE(self)->tp_name, 620 numfields == 0 ? "" : "either 0 or ", 621 numfields, numfields == 1 ? "" : "s"); 622 res = -1; 623 goto cleanup; 624 } 625 for (i = 0; i < PyTuple_GET_SIZE(args); i++) { 626 /* cannot be reached when fields is NULL */ 627 PyObject *name = PySequence_GetItem(fields, i); 628 if (!name) { 629 res = -1; 630 goto cleanup; 631 } 632 res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); 633 Py_DECREF(name); 634 if (res < 0) 635 goto cleanup; 636 } 637 } 638 if (kw) { 639 i = 0; /* needed by PyDict_Next */ 640 while (PyDict_Next(kw, &i, &key, &value)) { 641 res = PyObject_SetAttr(self, key, value); 642 if (res < 0) 643 goto cleanup; 644 } 645 } 646 cleanup: 647 Py_XDECREF(fields); 648 return res; 649 } 650 651 /* Pickling support */ 652 static PyObject * 653 ast_type_reduce(PyObject *self, PyObject *unused) 654 { 655 PyObject *res; 656 PyObject *dict = PyObject_GetAttrString(self, "__dict__"); 657 if (dict == NULL) { 658 if (PyErr_ExceptionMatches(PyExc_AttributeError)) 659 PyErr_Clear(); 660 else 661 return NULL; 662 } 663 if (dict) { 664 res = Py_BuildValue("O()O", Py_TYPE(self), dict); 665 Py_DECREF(dict); 666 return res; 667 } 668 return Py_BuildValue("O()", Py_TYPE(self)); 669 } 670 671 static PyMethodDef ast_type_methods[] = { 672 {"__reduce__", ast_type_reduce, METH_NOARGS, NULL}, 673 {NULL} 674 }; 675 676 static PyTypeObject AST_type = { 677 PyVarObject_HEAD_INIT(&PyType_Type, 0) 678 "_ast.AST", 679 sizeof(PyObject), 680 0, 681 0, /* tp_dealloc */ 682 0, /* tp_print */ 683 0, /* tp_getattr */ 684 0, /* tp_setattr */ 685 0, /* tp_compare */ 686 0, /* tp_repr */ 687 0, /* tp_as_number */ 688 0, /* tp_as_sequence */ 689 0, /* tp_as_mapping */ 690 0, /* tp_hash */ 691 0, /* tp_call */ 692 0, /* tp_str */ 693 PyObject_GenericGetAttr, /* tp_getattro */ 694 PyObject_GenericSetAttr, /* tp_setattro */ 695 0, /* tp_as_buffer */ 696 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ 697 0, /* tp_doc */ 698 0, /* tp_traverse */ 699 0, /* tp_clear */ 700 0, /* tp_richcompare */ 701 0, /* tp_weaklistoffset */ 702 0, /* tp_iter */ 703 0, /* tp_iternext */ 704 ast_type_methods, /* tp_methods */ 705 0, /* tp_members */ 706 0, /* tp_getset */ 707 0, /* tp_base */ 708 0, /* tp_dict */ 709 0, /* tp_descr_get */ 710 0, /* tp_descr_set */ 711 0, /* tp_dictoffset */ 712 (initproc)ast_type_init, /* tp_init */ 713 PyType_GenericAlloc, /* tp_alloc */ 714 PyType_GenericNew, /* tp_new */ 715 PyObject_Del, /* tp_free */ 716 }; 717 718 719 static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int num_fields) 720 { 721 PyObject *fnames, *result; 722 int i; 723 fnames = PyTuple_New(num_fields); 724 if (!fnames) return NULL; 725 for (i = 0; i < num_fields; i++) { 726 PyObject *field = PyString_FromString(fields[i]); 727 if (!field) { 728 Py_DECREF(fnames); 729 return NULL; 730 } 731 PyTuple_SET_ITEM(fnames, i, field); 732 } 733 result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){sOss}", 734 type, base, "_fields", fnames, "__module__", "_ast"); 735 Py_DECREF(fnames); 736 return (PyTypeObject*)result; 737 } 738 739 static int add_attributes(PyTypeObject* type, char**attrs, int num_fields) 740 { 741 int i, result; 742 PyObject *s, *l = PyTuple_New(num_fields); 743 if (!l) 744 return 0; 745 for (i = 0; i < num_fields; i++) { 746 s = PyString_FromString(attrs[i]); 747 if (!s) { 748 Py_DECREF(l); 749 return 0; 750 } 751 PyTuple_SET_ITEM(l, i, s); 752 } 753 result = PyObject_SetAttrString((PyObject*)type, "_attributes", l) >= 0; 754 Py_DECREF(l); 755 return result; 756 } 757 758 /* Conversion AST -> Python */ 759 760 static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*)) 761 { 762 int i, n = asdl_seq_LEN(seq); 763 PyObject *result = PyList_New(n); 764 PyObject *value; 765 if (!result) 766 return NULL; 767 for (i = 0; i < n; i++) { 768 value = func(asdl_seq_GET(seq, i)); 769 if (!value) { 770 Py_DECREF(result); 771 return NULL; 772 } 773 PyList_SET_ITEM(result, i, value); 774 } 775 return result; 776 } 777 778 static PyObject* ast2obj_object(void *o) 779 { 780 if (!o) 781 o = Py_None; 782 Py_INCREF((PyObject*)o); 783 return (PyObject*)o; 784 } 785 #define ast2obj_identifier ast2obj_object 786 #define ast2obj_string ast2obj_object 787 static PyObject* ast2obj_bool(bool b) 788 { 789 return PyBool_FromLong(b); 790 } 791 792 static PyObject* ast2obj_int(long b) 793 { 794 return PyInt_FromLong(b); 795 } 796 797 /* Conversion Python -> AST */ 798 799 static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena) 800 { 801 if (obj == Py_None) 802 obj = NULL; 803 if (obj) 804 PyArena_AddPyObject(arena, obj); 805 Py_XINCREF(obj); 806 *out = obj; 807 return 0; 808 } 809 810 static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena) 811 { 812 if (!PyString_CheckExact(obj) && obj != Py_None) { 813 PyErr_Format(PyExc_TypeError, 814 "AST identifier must be of type str"); 815 return 1; 816 } 817 return obj2ast_object(obj, out, arena); 818 } 819 820 static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena) 821 { 822 if (!PyString_CheckExact(obj) && !PyUnicode_CheckExact(obj)) { 823 PyErr_SetString(PyExc_TypeError, 824 "AST string must be of type str or unicode"); 825 return 1; 826 } 827 return obj2ast_object(obj, out, arena); 828 } 829 830 static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) 831 { 832 int i; 833 if (!PyInt_Check(obj) && !PyLong_Check(obj)) { 834 PyObject *s = PyObject_Repr(obj); 835 if (s == NULL) return 1; 836 PyErr_Format(PyExc_ValueError, "invalid integer value: %.400s", 837 PyString_AS_STRING(s)); 838 Py_DECREF(s); 839 return 1; 840 } 841 842 i = (int)PyLong_AsLong(obj); 843 if (i == -1 && PyErr_Occurred()) 844 return 1; 845 *out = i; 846 return 0; 847 } 848 849 static int obj2ast_bool(PyObject* obj, bool* out, PyArena* arena) 850 { 851 if (!PyBool_Check(obj)) { 852 PyObject *s = PyObject_Repr(obj); 853 if (s == NULL) return 1; 854 PyErr_Format(PyExc_ValueError, "invalid boolean value: %.400s", 855 PyString_AS_STRING(s)); 856 Py_DECREF(s); 857 return 1; 858 } 859 860 *out = (obj == Py_True); 861 return 0; 862 } 863 864 static int add_ast_fields(void) 865 { 866 PyObject *empty_tuple, *d; 867 if (PyType_Ready(&AST_type) < 0) 868 return -1; 869 d = AST_type.tp_dict; 870 empty_tuple = PyTuple_New(0); 871 if (!empty_tuple || 872 PyDict_SetItemString(d, "_fields", empty_tuple) < 0 || 873 PyDict_SetItemString(d, "_attributes", empty_tuple) < 0) { 874 Py_XDECREF(empty_tuple); 875 return -1; 876 } 877 Py_DECREF(empty_tuple); 878 return 0; 879 } 880 881 """, 0, reflow=False) 882 883 self.emit("static int init_types(void)",0) 884 self.emit("{", 0) 885 self.emit("static int initialized;", 1) 886 self.emit("if (initialized) return 1;", 1) 887 self.emit("if (add_ast_fields() < 0) return 0;", 1) 888 for dfn in mod.dfns: 889 self.visit(dfn) 890 self.emit("initialized = 1;", 1) 891 self.emit("return 1;", 1); 892 self.emit("}", 0) 893 894 def visitProduct(self, prod, name): 895 if prod.fields: 896 fields = name.value+"_fields" 897 else: 898 fields = "NULL" 899 self.emit('%s_type = make_type("%s", &AST_type, %s, %d);' % 900 (name, name, fields, len(prod.fields)), 1) 901 self.emit("if (!%s_type) return 0;" % name, 1) 902 903 def visitSum(self, sum, name): 904 self.emit('%s_type = make_type("%s", &AST_type, NULL, 0);' % 905 (name, name), 1) 906 self.emit("if (!%s_type) return 0;" % name, 1) 907 if sum.attributes: 908 self.emit("if (!add_attributes(%s_type, %s_attributes, %d)) return 0;" % 909 (name, name, len(sum.attributes)), 1) 910 else: 911 self.emit("if (!add_attributes(%s_type, NULL, 0)) return 0;" % name, 1) 912 simple = is_simple(sum) 913 for t in sum.types: 914 self.visitConstructor(t, name, simple) 915 916 def visitConstructor(self, cons, name, simple): 917 if cons.fields: 918 fields = cons.name.value+"_fields" 919 else: 920 fields = "NULL" 921 self.emit('%s_type = make_type("%s", %s_type, %s, %d);' % 922 (cons.name, cons.name, name, fields, len(cons.fields)), 1) 923 self.emit("if (!%s_type) return 0;" % cons.name, 1) 924 if simple: 925 self.emit("%s_singleton = PyType_GenericNew(%s_type, NULL, NULL);" % 926 (cons.name, cons.name), 1) 927 self.emit("if (!%s_singleton) return 0;" % cons.name, 1) 928 929 930 class ASTModuleVisitor(PickleVisitor): 931 932 def visitModule(self, mod): 933 self.emit("PyMODINIT_FUNC", 0) 934 self.emit("init_ast(void)", 0) 935 self.emit("{", 0) 936 self.emit("PyObject *m, *d;", 1) 937 self.emit("if (!init_types()) return;", 1) 938 self.emit('m = Py_InitModule3("_ast", NULL, NULL);', 1) 939 self.emit("if (!m) return;", 1) 940 self.emit("d = PyModule_GetDict(m);", 1) 941 self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return;', 1) 942 self.emit('if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0)', 1) 943 self.emit("return;", 2) 944 # Value of version: "$Revision$" 945 self.emit('if (PyModule_AddStringConstant(m, "__version__", "%s") < 0)' 946 % mod.version, 1) 947 self.emit("return;", 2) 948 for dfn in mod.dfns: 949 self.visit(dfn) 950 self.emit("}", 0) 951 952 def visitProduct(self, prod, name): 953 self.addObj(name) 954 955 def visitSum(self, sum, name): 956 self.addObj(name) 957 for t in sum.types: 958 self.visitConstructor(t, name) 959 960 def visitConstructor(self, cons, name): 961 self.addObj(cons.name) 962 963 def addObj(self, name): 964 self.emit('if (PyDict_SetItemString(d, "%s", (PyObject*)%s_type) < 0) return;' % (name, name), 1) 965 966 967 _SPECIALIZED_SEQUENCES = ('stmt', 'expr') 968 969 def find_sequence(fields, doing_specialization): 970 """Return True if any field uses a sequence.""" 971 for f in fields: 972 if f.seq: 973 if not doing_specialization: 974 return True 975 if str(f.type) not in _SPECIALIZED_SEQUENCES: 976 return True 977 return False 978 979 def has_sequence(types, doing_specialization): 980 for t in types: 981 if find_sequence(t.fields, doing_specialization): 982 return True 983 return False 984 985 986 class StaticVisitor(PickleVisitor): 987 CODE = '''Very simple, always emit this static code. Override CODE''' 988 989 def visit(self, object): 990 self.emit(self.CODE, 0, reflow=False) 991 992 993 class ObjVisitor(PickleVisitor): 994 995 def func_begin(self, name): 996 ctype = get_c_type(name) 997 self.emit("PyObject*", 0) 998 self.emit("ast2obj_%s(void* _o)" % (name), 0) 999 self.emit("{", 0) 1000 self.emit("%s o = (%s)_o;" % (ctype, ctype), 1) 1001 self.emit("PyObject *result = NULL, *value = NULL;", 1) 1002 self.emit('if (!o) {', 1) 1003 self.emit("Py_INCREF(Py_None);", 2) 1004 self.emit('return Py_None;', 2) 1005 self.emit("}", 1) 1006 self.emit('', 0) 1007 1008 def func_end(self): 1009 self.emit("return result;", 1) 1010 self.emit("failed:", 0) 1011 self.emit("Py_XDECREF(value);", 1) 1012 self.emit("Py_XDECREF(result);", 1) 1013 self.emit("return NULL;", 1) 1014 self.emit("}", 0) 1015 self.emit("", 0) 1016 1017 def visitSum(self, sum, name): 1018 if is_simple(sum): 1019 self.simpleSum(sum, name) 1020 return 1021 self.func_begin(name) 1022 self.emit("switch (o->kind) {", 1) 1023 for i in range(len(sum.types)): 1024 t = sum.types[i] 1025 self.visitConstructor(t, i + 1, name) 1026 self.emit("}", 1) 1027 for a in sum.attributes: 1028 self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1) 1029 self.emit("if (!value) goto failed;", 1) 1030 self.emit('if (PyObject_SetAttrString(result, "%s", value) < 0)' % a.name, 1) 1031 self.emit('goto failed;', 2) 1032 self.emit('Py_DECREF(value);', 1) 1033 self.func_end() 1034 1035 def simpleSum(self, sum, name): 1036 self.emit("PyObject* ast2obj_%s(%s_ty o)" % (name, name), 0) 1037 self.emit("{", 0) 1038 self.emit("switch(o) {", 1) 1039 for t in sum.types: 1040 self.emit("case %s:" % t.name, 2) 1041 self.emit("Py_INCREF(%s_singleton);" % t.name, 3) 1042 self.emit("return %s_singleton;" % t.name, 3) 1043 self.emit("default:", 2) 1044 self.emit('/* should never happen, but just in case ... */', 3) 1045 code = "PyErr_Format(PyExc_SystemError, \"unknown %s found\");" % name 1046 self.emit(code, 3, reflow=False) 1047 self.emit("return NULL;", 3) 1048 self.emit("}", 1) 1049 self.emit("}", 0) 1050 1051 def visitProduct(self, prod, name): 1052 self.func_begin(name) 1053 self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % name, 1); 1054 self.emit("if (!result) return NULL;", 1) 1055 for field in prod.fields: 1056 self.visitField(field, name, 1, True) 1057 self.func_end() 1058 1059 def visitConstructor(self, cons, enum, name): 1060 self.emit("case %s_kind:" % cons.name, 1) 1061 self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % cons.name, 2); 1062 self.emit("if (!result) goto failed;", 2) 1063 for f in cons.fields: 1064 self.visitField(f, cons.name, 2, False) 1065 self.emit("break;", 2) 1066 1067 def visitField(self, field, name, depth, product): 1068 def emit(s, d): 1069 self.emit(s, depth + d) 1070 if product: 1071 value = "o->%s" % field.name 1072 else: 1073 value = "o->v.%s.%s" % (name, field.name) 1074 self.set(field, value, depth) 1075 emit("if (!value) goto failed;", 0) 1076 emit('if (PyObject_SetAttrString(result, "%s", value) == -1)' % field.name, 0) 1077 emit("goto failed;", 1) 1078 emit("Py_DECREF(value);", 0) 1079 1080 def emitSeq(self, field, value, depth, emit): 1081 emit("seq = %s;" % value, 0) 1082 emit("n = asdl_seq_LEN(seq);", 0) 1083 emit("value = PyList_New(n);", 0) 1084 emit("if (!value) goto failed;", 0) 1085 emit("for (i = 0; i < n; i++) {", 0) 1086 self.set("value", field, "asdl_seq_GET(seq, i)", depth + 1) 1087 emit("if (!value1) goto failed;", 1) 1088 emit("PyList_SET_ITEM(value, i, value1);", 1) 1089 emit("value1 = NULL;", 1) 1090 emit("}", 0) 1091 1092 def set(self, field, value, depth): 1093 if field.seq: 1094 # XXX should really check for is_simple, but that requires a symbol table 1095 if field.type.value == "cmpop": 1096 # While the sequence elements are stored as void*, 1097 # ast2obj_cmpop expects an enum 1098 self.emit("{", depth) 1099 self.emit("int i, n = asdl_seq_LEN(%s);" % value, depth+1) 1100 self.emit("value = PyList_New(n);", depth+1) 1101 self.emit("if (!value) goto failed;", depth+1) 1102 self.emit("for(i = 0; i < n; i++)", depth+1) 1103 # This cannot fail, so no need for error handling 1104 self.emit("PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(%s, i)));" % value, 1105 depth+2, reflow=False) 1106 self.emit("}", depth) 1107 else: 1108 self.emit("value = ast2obj_list(%s, ast2obj_%s);" % (value, field.type), depth) 1109 else: 1110 ctype = get_c_type(field.type) 1111 self.emit("value = ast2obj_%s(%s);" % (field.type, value), depth, reflow=False) 1112 1113 1114 class PartingShots(StaticVisitor): 1115 1116 CODE = """ 1117 PyObject* PyAST_mod2obj(mod_ty t) 1118 { 1119 init_types(); 1120 return ast2obj_mod(t); 1121 } 1122 1123 /* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */ 1124 mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) 1125 { 1126 mod_ty res; 1127 PyObject *req_type[3]; 1128 char *req_name[3]; 1129 int isinstance; 1130 1131 req_type[0] = (PyObject*)Module_type; 1132 req_type[1] = (PyObject*)Expression_type; 1133 req_type[2] = (PyObject*)Interactive_type; 1134 1135 req_name[0] = "Module"; 1136 req_name[1] = "Expression"; 1137 req_name[2] = "Interactive"; 1138 1139 assert(0 <= mode && mode <= 2); 1140 1141 init_types(); 1142 1143 isinstance = PyObject_IsInstance(ast, req_type[mode]); 1144 if (isinstance == -1) 1145 return NULL; 1146 if (!isinstance) { 1147 PyErr_Format(PyExc_TypeError, "expected %s node, got %.400s", 1148 req_name[mode], Py_TYPE(ast)->tp_name); 1149 return NULL; 1150 } 1151 if (obj2ast_mod(ast, &res, arena) != 0) 1152 return NULL; 1153 else 1154 return res; 1155 } 1156 1157 int PyAST_Check(PyObject* obj) 1158 { 1159 init_types(); 1160 return PyObject_IsInstance(obj, (PyObject*)&AST_type); 1161 } 1162 """ 1163 1164 class ChainOfVisitors: 1165 def __init__(self, *visitors): 1166 self.visitors = visitors 1167 1168 def visit(self, object): 1169 for v in self.visitors: 1170 v.visit(object) 1171 v.emit("", 0) 1172 1173 common_msg = "/* File automatically generated by %s. */\n\n" 1174 1175 c_file_msg = """ 1176 /* 1177 __version__ %s. 1178 1179 This module must be committed separately after each AST grammar change; 1180 The __version__ number is set to the revision number of the commit 1181 containing the grammar change. 1182 */ 1183 1184 """ 1185 1186 def main(srcfile): 1187 argv0 = sys.argv[0] 1188 components = argv0.split(os.sep) 1189 argv0 = os.sep.join(components[-2:]) 1190 auto_gen_msg = common_msg % argv0 1191 mod = asdl.parse(srcfile) 1192 mod.version = "82160" 1193 if not asdl.check(mod): 1194 sys.exit(1) 1195 if INC_DIR: 1196 p = "%s/%s-ast.h" % (INC_DIR, mod.name) 1197 f = open(p, "wb") 1198 f.write(auto_gen_msg) 1199 f.write('#include "asdl.h"\n\n') 1200 c = ChainOfVisitors(TypeDefVisitor(f), 1201 StructVisitor(f), 1202 PrototypeVisitor(f), 1203 ) 1204 c.visit(mod) 1205 f.write("PyObject* PyAST_mod2obj(mod_ty t);\n") 1206 f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n") 1207 f.write("int PyAST_Check(PyObject* obj);\n") 1208 f.close() 1209 1210 if SRC_DIR: 1211 p = os.path.join(SRC_DIR, str(mod.name) + "-ast.c") 1212 f = open(p, "wb") 1213 f.write(auto_gen_msg) 1214 f.write(c_file_msg % mod.version) 1215 f.write('#include "Python.h"\n') 1216 f.write('#include "%s-ast.h"\n' % mod.name) 1217 f.write('\n') 1218 f.write("static PyTypeObject AST_type;\n") 1219 v = ChainOfVisitors( 1220 PyTypesDeclareVisitor(f), 1221 PyTypesVisitor(f), 1222 Obj2ModPrototypeVisitor(f), 1223 FunctionVisitor(f), 1224 ObjVisitor(f), 1225 Obj2ModVisitor(f), 1226 ASTModuleVisitor(f), 1227 PartingShots(f), 1228 ) 1229 v.visit(mod) 1230 f.close() 1231 1232 if __name__ == "__main__": 1233 import sys 1234 import getopt 1235 1236 INC_DIR = '' 1237 SRC_DIR = '' 1238 opts, args = getopt.getopt(sys.argv[1:], "h:c:") 1239 if len(opts) != 1: 1240 print "Must specify exactly one output file" 1241 sys.exit(1) 1242 for o, v in opts: 1243 if o == '-h': 1244 INC_DIR = v 1245 if o == '-c': 1246 SRC_DIR = v 1247 if len(args) != 1: 1248 print "Must specify single input file" 1249 sys.exit(1) 1250 main(args[0]) 1251
