1 /** @file 2 Write Python objects to files and read them back. 3 This is intended for writing and reading compiled Python code only; 4 a true persistent storage facility would be much harder, since 5 it would have to take circular links and sharing into account. 6 7 Copyright (c) 2011, Intel Corporation. All rights reserved.<BR> 8 This program and the accompanying materials are licensed and made available under 9 the terms and conditions of the BSD License that accompanies this distribution. 10 The full text of the license may be found at 11 http://opensource.org/licenses/bsd-license. 12 13 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, 14 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. 15 **/ 16 17 #define PY_SSIZE_T_CLEAN 18 19 #include "Python.h" 20 #include "longintrepr.h" 21 #include "code.h" 22 #include "marshal.h" 23 24 #ifndef ABS 25 #define ABS(x) ((x) < 0 ? -(x) : (x)) 26 #endif 27 28 /* High water mark to determine when the marshalled object is dangerously deep 29 * and risks coring the interpreter. When the object stack gets this deep, 30 * raise an exception instead of continuing. 31 */ 32 #define MAX_MARSHAL_STACK_DEPTH 2000 33 34 #define TYPE_NULL '0' 35 #define TYPE_NONE 'N' 36 #define TYPE_FALSE 'F' 37 #define TYPE_TRUE 'T' 38 #define TYPE_STOPITER 'S' 39 #define TYPE_ELLIPSIS '.' 40 #define TYPE_INT 'i' 41 #define TYPE_INT64 'I' 42 #define TYPE_FLOAT 'f' 43 #define TYPE_BINARY_FLOAT 'g' 44 #define TYPE_COMPLEX 'x' 45 #define TYPE_BINARY_COMPLEX 'y' 46 #define TYPE_LONG 'l' 47 #define TYPE_STRING 's' 48 #define TYPE_INTERNED 't' 49 #define TYPE_STRINGREF 'R' 50 #define TYPE_TUPLE '(' 51 #define TYPE_LIST '[' 52 #define TYPE_DICT '{' 53 #define TYPE_CODE 'c' 54 #define TYPE_UNICODE 'u' 55 #define TYPE_UNKNOWN '?' 56 #define TYPE_SET '<' 57 #define TYPE_FROZENSET '>' 58 59 #define WFERR_OK 0 60 #define WFERR_UNMARSHALLABLE 1 61 #define WFERR_NESTEDTOODEEP 2 62 #define WFERR_NOMEMORY 3 63 64 typedef struct { 65 FILE *fp; 66 int error; /* see WFERR_* values */ 67 int depth; 68 /* If fp == NULL, the following are valid: */ 69 PyObject *str; 70 char *ptr; 71 char *end; 72 PyObject *strings; /* dict on marshal, list on unmarshal */ 73 int version; 74 } WFILE; 75 76 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \ 77 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \ 78 else w_more(c, p) 79 80 static void 81 w_more(int c, WFILE *p) 82 { 83 Py_ssize_t size, newsize; 84 if (p->str == NULL) 85 return; /* An error already occurred */ 86 size = PyString_Size(p->str); 87 newsize = size + size + 1024; 88 if (newsize > 32*1024*1024) { 89 newsize = size + (size >> 3); /* 12.5% overallocation */ 90 } 91 if (_PyString_Resize(&p->str, newsize) != 0) { 92 p->ptr = p->end = NULL; 93 } 94 else { 95 p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size; 96 p->end = 97 PyString_AS_STRING((PyStringObject *)p->str) + newsize; 98 *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char); 99 } 100 } 101 102 static void 103 w_string(char *s, int n, WFILE *p) 104 { 105 if (p->fp != NULL) { 106 fwrite(s, 1, n, p->fp); 107 } 108 else { 109 while (--n >= 0) { 110 w_byte(*s, p); 111 s++; 112 } 113 } 114 } 115 116 static void 117 w_short(int x, WFILE *p) 118 { 119 w_byte((char)( x & 0xff), p); 120 w_byte((char)((x>> 8) & 0xff), p); 121 } 122 123 static void 124 w_long(long x, WFILE *p) 125 { 126 w_byte((char)( x & 0xff), p); 127 w_byte((char)((x>> 8) & 0xff), p); 128 w_byte((char)((x>>16) & 0xff), p); 129 w_byte((char)((x>>24) & 0xff), p); 130 } 131 132 #if SIZEOF_LONG > 4 133 static void 134 w_long64(long x, WFILE *p) 135 { 136 w_long(x, p); 137 w_long(x>>32, p); 138 } 139 #endif 140 141 /* We assume that Python longs are stored internally in base some power of 142 2**15; for the sake of portability we'll always read and write them in base 143 exactly 2**15. */ 144 145 #define PyLong_MARSHAL_SHIFT 15 146 #define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT) 147 #define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1) 148 #if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0 149 #error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT" 150 #endif 151 #define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT) 152 153 static void 154 w_PyLong(const PyLongObject *ob, WFILE *p) 155 { 156 Py_ssize_t i, j, n, l; 157 digit d; 158 159 w_byte(TYPE_LONG, p); 160 if (Py_SIZE(ob) == 0) { 161 w_long((long)0, p); 162 return; 163 } 164 165 /* set l to number of base PyLong_MARSHAL_BASE digits */ 166 n = ABS(Py_SIZE(ob)); 167 l = (n-1) * PyLong_MARSHAL_RATIO; 168 d = ob->ob_digit[n-1]; 169 assert(d != 0); /* a PyLong is always normalized */ 170 do { 171 d >>= PyLong_MARSHAL_SHIFT; 172 l++; 173 } while (d != 0); 174 w_long((long)(Py_SIZE(ob) > 0 ? l : -l), p); 175 176 for (i=0; i < n-1; i++) { 177 d = ob->ob_digit[i]; 178 for (j=0; j < PyLong_MARSHAL_RATIO; j++) { 179 w_short(d & PyLong_MARSHAL_MASK, p); 180 d >>= PyLong_MARSHAL_SHIFT; 181 } 182 assert (d == 0); 183 } 184 d = ob->ob_digit[n-1]; 185 do { 186 w_short(d & PyLong_MARSHAL_MASK, p); 187 d >>= PyLong_MARSHAL_SHIFT; 188 } while (d != 0); 189 } 190 191 static void 192 w_object(PyObject *v, WFILE *p) 193 { 194 Py_ssize_t i, n; 195 196 p->depth++; 197 198 if (p->depth > MAX_MARSHAL_STACK_DEPTH) { 199 p->error = WFERR_NESTEDTOODEEP; 200 } 201 else if (v == NULL) { 202 w_byte(TYPE_NULL, p); 203 } 204 else if (v == Py_None) { 205 w_byte(TYPE_NONE, p); 206 } 207 else if (v == PyExc_StopIteration) { 208 w_byte(TYPE_STOPITER, p); 209 } 210 else if (v == Py_Ellipsis) { 211 w_byte(TYPE_ELLIPSIS, p); 212 } 213 else if (v == Py_False) { 214 w_byte(TYPE_FALSE, p); 215 } 216 else if (v == Py_True) { 217 w_byte(TYPE_TRUE, p); 218 } 219 else if (PyInt_CheckExact(v)) { 220 long x = PyInt_AS_LONG((PyIntObject *)v); 221 #if SIZEOF_LONG > 4 222 long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31); 223 if (y && y != -1) { 224 w_byte(TYPE_INT64, p); 225 w_long64(x, p); 226 } 227 else 228 #endif 229 { 230 w_byte(TYPE_INT, p); 231 w_long(x, p); 232 } 233 } 234 else if (PyLong_CheckExact(v)) { 235 PyLongObject *ob = (PyLongObject *)v; 236 w_PyLong(ob, p); 237 } 238 else if (PyFloat_CheckExact(v)) { 239 if (p->version > 1) { 240 unsigned char buf[8]; 241 if (_PyFloat_Pack8(PyFloat_AsDouble(v), 242 buf, 1) < 0) { 243 p->error = WFERR_UNMARSHALLABLE; 244 return; 245 } 246 w_byte(TYPE_BINARY_FLOAT, p); 247 w_string((char*)buf, 8, p); 248 } 249 else { 250 char *buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v), 251 'g', 17, 0, NULL); 252 if (!buf) { 253 p->error = WFERR_NOMEMORY; 254 return; 255 } 256 n = strlen(buf); 257 w_byte(TYPE_FLOAT, p); 258 w_byte((int)n, p); 259 w_string(buf, (int)n, p); 260 PyMem_Free(buf); 261 } 262 } 263 #ifndef WITHOUT_COMPLEX 264 else if (PyComplex_CheckExact(v)) { 265 if (p->version > 1) { 266 unsigned char buf[8]; 267 if (_PyFloat_Pack8(PyComplex_RealAsDouble(v), 268 buf, 1) < 0) { 269 p->error = WFERR_UNMARSHALLABLE; 270 return; 271 } 272 w_byte(TYPE_BINARY_COMPLEX, p); 273 w_string((char*)buf, 8, p); 274 if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v), 275 buf, 1) < 0) { 276 p->error = WFERR_UNMARSHALLABLE; 277 return; 278 } 279 w_string((char*)buf, 8, p); 280 } 281 else { 282 char *buf; 283 w_byte(TYPE_COMPLEX, p); 284 buf = PyOS_double_to_string(PyComplex_RealAsDouble(v), 285 'g', 17, 0, NULL); 286 if (!buf) { 287 p->error = WFERR_NOMEMORY; 288 return; 289 } 290 n = strlen(buf); 291 w_byte((int)n, p); 292 w_string(buf, (int)n, p); 293 PyMem_Free(buf); 294 buf = PyOS_double_to_string(PyComplex_ImagAsDouble(v), 295 'g', 17, 0, NULL); 296 if (!buf) { 297 p->error = WFERR_NOMEMORY; 298 return; 299 } 300 n = strlen(buf); 301 w_byte((int)n, p); 302 w_string(buf, (int)n, p); 303 PyMem_Free(buf); 304 } 305 } 306 #endif 307 else if (PyString_CheckExact(v)) { 308 if (p->strings && PyString_CHECK_INTERNED(v)) { 309 PyObject *o = PyDict_GetItem(p->strings, v); 310 if (o) { 311 long w = PyInt_AsLong(o); 312 w_byte(TYPE_STRINGREF, p); 313 w_long(w, p); 314 goto exit; 315 } 316 else { 317 int ok; 318 o = PyInt_FromSsize_t(PyDict_Size(p->strings)); 319 ok = o && 320 PyDict_SetItem(p->strings, v, o) >= 0; 321 Py_XDECREF(o); 322 if (!ok) { 323 p->depth--; 324 p->error = WFERR_UNMARSHALLABLE; 325 return; 326 } 327 w_byte(TYPE_INTERNED, p); 328 } 329 } 330 else { 331 w_byte(TYPE_STRING, p); 332 } 333 n = PyString_GET_SIZE(v); 334 if (n > INT_MAX) { 335 /* huge strings are not supported */ 336 p->depth--; 337 p->error = WFERR_UNMARSHALLABLE; 338 return; 339 } 340 w_long((long)n, p); 341 w_string(PyString_AS_STRING(v), (int)n, p); 342 } 343 #ifdef Py_USING_UNICODE 344 else if (PyUnicode_CheckExact(v)) { 345 PyObject *utf8; 346 utf8 = PyUnicode_AsUTF8String(v); 347 if (utf8 == NULL) { 348 p->depth--; 349 p->error = WFERR_UNMARSHALLABLE; 350 return; 351 } 352 w_byte(TYPE_UNICODE, p); 353 n = PyString_GET_SIZE(utf8); 354 if (n > INT_MAX) { 355 p->depth--; 356 p->error = WFERR_UNMARSHALLABLE; 357 return; 358 } 359 w_long((long)n, p); 360 w_string(PyString_AS_STRING(utf8), (int)n, p); 361 Py_DECREF(utf8); 362 } 363 #endif 364 else if (PyTuple_CheckExact(v)) { 365 w_byte(TYPE_TUPLE, p); 366 n = PyTuple_Size(v); 367 w_long((long)n, p); 368 for (i = 0; i < n; i++) { 369 w_object(PyTuple_GET_ITEM(v, i), p); 370 } 371 } 372 else if (PyList_CheckExact(v)) { 373 w_byte(TYPE_LIST, p); 374 n = PyList_GET_SIZE(v); 375 w_long((long)n, p); 376 for (i = 0; i < n; i++) { 377 w_object(PyList_GET_ITEM(v, i), p); 378 } 379 } 380 else if (PyDict_CheckExact(v)) { 381 Py_ssize_t pos; 382 PyObject *key, *value; 383 w_byte(TYPE_DICT, p); 384 /* This one is NULL object terminated! */ 385 pos = 0; 386 while (PyDict_Next(v, &pos, &key, &value)) { 387 w_object(key, p); 388 w_object(value, p); 389 } 390 w_object((PyObject *)NULL, p); 391 } 392 else if (PyAnySet_CheckExact(v)) { 393 PyObject *value, *it; 394 395 if (PyObject_TypeCheck(v, &PySet_Type)) 396 w_byte(TYPE_SET, p); 397 else 398 w_byte(TYPE_FROZENSET, p); 399 n = PyObject_Size(v); 400 if (n == -1) { 401 p->depth--; 402 p->error = WFERR_UNMARSHALLABLE; 403 return; 404 } 405 w_long((long)n, p); 406 it = PyObject_GetIter(v); 407 if (it == NULL) { 408 p->depth--; 409 p->error = WFERR_UNMARSHALLABLE; 410 return; 411 } 412 while ((value = PyIter_Next(it)) != NULL) { 413 w_object(value, p); 414 Py_DECREF(value); 415 } 416 Py_DECREF(it); 417 if (PyErr_Occurred()) { 418 p->depth--; 419 p->error = WFERR_UNMARSHALLABLE; 420 return; 421 } 422 } 423 else if (PyCode_Check(v)) { 424 PyCodeObject *co = (PyCodeObject *)v; 425 w_byte(TYPE_CODE, p); 426 w_long(co->co_argcount, p); 427 w_long(co->co_nlocals, p); 428 w_long(co->co_stacksize, p); 429 w_long(co->co_flags, p); 430 w_object(co->co_code, p); 431 w_object(co->co_consts, p); 432 w_object(co->co_names, p); 433 w_object(co->co_varnames, p); 434 w_object(co->co_freevars, p); 435 w_object(co->co_cellvars, p); 436 w_object(co->co_filename, p); 437 w_object(co->co_name, p); 438 w_long(co->co_firstlineno, p); 439 w_object(co->co_lnotab, p); 440 } 441 else if (PyObject_CheckReadBuffer(v)) { 442 /* Write unknown buffer-style objects as a string */ 443 char *s; 444 PyBufferProcs *pb = v->ob_type->tp_as_buffer; 445 w_byte(TYPE_STRING, p); 446 n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s); 447 if (n > INT_MAX) { 448 p->depth--; 449 p->error = WFERR_UNMARSHALLABLE; 450 return; 451 } 452 w_long((long)n, p); 453 w_string(s, (int)n, p); 454 } 455 else { 456 w_byte(TYPE_UNKNOWN, p); 457 p->error = WFERR_UNMARSHALLABLE; 458 } 459 exit: 460 p->depth--; 461 } 462 463 /* version currently has no effect for writing longs. */ 464 void 465 PyMarshal_WriteLongToFile(long x, FILE *fp, int version) 466 { 467 WFILE wf; 468 wf.fp = fp; 469 wf.error = WFERR_OK; 470 wf.depth = 0; 471 wf.strings = NULL; 472 wf.version = version; 473 w_long(x, &wf); 474 } 475 476 void 477 PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version) 478 { 479 WFILE wf; 480 wf.fp = fp; 481 wf.error = WFERR_OK; 482 wf.depth = 0; 483 wf.strings = (version > 0) ? PyDict_New() : NULL; 484 wf.version = version; 485 w_object(x, &wf); 486 Py_XDECREF(wf.strings); 487 } 488 489 typedef WFILE RFILE; /* Same struct with different invariants */ 490 491 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF) 492 493 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p)) 494 495 static int 496 r_string(char *s, int n, RFILE *p) 497 { 498 if (p->fp != NULL) 499 /* The result fits into int because it must be <=n. */ 500 return (int)fread(s, 1, n, p->fp); 501 if (p->end - p->ptr < n) 502 n = (int)(p->end - p->ptr); 503 memcpy(s, p->ptr, n); 504 p->ptr += n; 505 return n; 506 } 507 508 static int 509 r_short(RFILE *p) 510 { 511 register short x; 512 x = r_byte(p); 513 x |= r_byte(p) << 8; 514 /* Sign-extension, in case short greater than 16 bits */ 515 x |= -(x & 0x8000); 516 return x; 517 } 518 519 static long 520 r_long(RFILE *p) 521 { 522 register long x; 523 register FILE *fp = p->fp; 524 if (fp) { 525 x = getc(fp); 526 x |= (long)getc(fp) << 8; 527 x |= (long)getc(fp) << 16; 528 x |= (long)getc(fp) << 24; 529 } 530 else { 531 x = rs_byte(p); 532 x |= (long)rs_byte(p) << 8; 533 x |= (long)rs_byte(p) << 16; 534 x |= (long)rs_byte(p) << 24; 535 } 536 #if SIZEOF_LONG > 4 537 /* Sign extension for 64-bit machines */ 538 x |= -(x & 0x80000000L); 539 #endif 540 return x; 541 } 542 543 /* r_long64 deals with the TYPE_INT64 code. On a machine with 544 sizeof(long) > 4, it returns a Python int object, else a Python long 545 object. Note that w_long64 writes out TYPE_INT if 32 bits is enough, 546 so there's no inefficiency here in returning a PyLong on 32-bit boxes 547 for everything written via TYPE_INT64 (i.e., if an int is written via 548 TYPE_INT64, it *needs* more than 32 bits). 549 */ 550 static PyObject * 551 r_long64(RFILE *p) 552 { 553 long lo4 = r_long(p); 554 long hi4 = r_long(p); 555 #if SIZEOF_LONG > 4 556 long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL); 557 return PyInt_FromLong(x); 558 #else 559 unsigned char buf[8]; 560 int one = 1; 561 int is_little_endian = (int)*(char*)&one; 562 if (is_little_endian) { 563 memcpy(buf, &lo4, 4); 564 memcpy(buf+4, &hi4, 4); 565 } 566 else { 567 memcpy(buf, &hi4, 4); 568 memcpy(buf+4, &lo4, 4); 569 } 570 return _PyLong_FromByteArray(buf, 8, is_little_endian, 1); 571 #endif 572 } 573 574 static PyObject * 575 r_PyLong(RFILE *p) 576 { 577 PyLongObject *ob; 578 int size, i, j, md, shorts_in_top_digit; 579 long n; 580 digit d; 581 582 n = r_long(p); 583 if (n == 0) 584 return (PyObject *)_PyLong_New(0); 585 if (n < -INT_MAX || n > INT_MAX) { 586 PyErr_SetString(PyExc_ValueError, 587 "bad marshal data (long size out of range)"); 588 return NULL; 589 } 590 591 size = 1 + (ABS(n) - 1) / PyLong_MARSHAL_RATIO; 592 shorts_in_top_digit = 1 + (ABS(n) - 1) % PyLong_MARSHAL_RATIO; 593 ob = _PyLong_New(size); 594 if (ob == NULL) 595 return NULL; 596 Py_SIZE(ob) = n > 0 ? size : -size; 597 598 for (i = 0; i < size-1; i++) { 599 d = 0; 600 for (j=0; j < PyLong_MARSHAL_RATIO; j++) { 601 md = r_short(p); 602 if (md < 0 || md > PyLong_MARSHAL_BASE) 603 goto bad_digit; 604 d += (digit)md << j*PyLong_MARSHAL_SHIFT; 605 } 606 ob->ob_digit[i] = d; 607 } 608 d = 0; 609 for (j=0; j < shorts_in_top_digit; j++) { 610 md = r_short(p); 611 if (md < 0 || md > PyLong_MARSHAL_BASE) 612 goto bad_digit; 613 /* topmost marshal digit should be nonzero */ 614 if (md == 0 && j == shorts_in_top_digit - 1) { 615 Py_DECREF(ob); 616 PyErr_SetString(PyExc_ValueError, 617 "bad marshal data (unnormalized long data)"); 618 return NULL; 619 } 620 d += (digit)md << j*PyLong_MARSHAL_SHIFT; 621 } 622 /* top digit should be nonzero, else the resulting PyLong won't be 623 normalized */ 624 ob->ob_digit[size-1] = d; 625 return (PyObject *)ob; 626 bad_digit: 627 Py_DECREF(ob); 628 PyErr_SetString(PyExc_ValueError, 629 "bad marshal data (digit out of range in long)"); 630 return NULL; 631 } 632 633 634 static PyObject * 635 r_object(RFILE *p) 636 { 637 /* NULL is a valid return value, it does not necessarily means that 638 an exception is set. */ 639 PyObject *v, *v2; 640 long i, n; 641 int type = r_byte(p); 642 PyObject *retval; 643 644 p->depth++; 645 646 if (p->depth > MAX_MARSHAL_STACK_DEPTH) { 647 p->depth--; 648 PyErr_SetString(PyExc_ValueError, "recursion limit exceeded"); 649 return NULL; 650 } 651 652 switch (type) { 653 654 case EOF: 655 PyErr_SetString(PyExc_EOFError, 656 "EOF read where object expected"); 657 retval = NULL; 658 break; 659 660 case TYPE_NULL: 661 retval = NULL; 662 break; 663 664 case TYPE_NONE: 665 Py_INCREF(Py_None); 666 retval = Py_None; 667 break; 668 669 case TYPE_STOPITER: 670 Py_INCREF(PyExc_StopIteration); 671 retval = PyExc_StopIteration; 672 break; 673 674 case TYPE_ELLIPSIS: 675 Py_INCREF(Py_Ellipsis); 676 retval = Py_Ellipsis; 677 break; 678 679 case TYPE_FALSE: 680 Py_INCREF(Py_False); 681 retval = Py_False; 682 break; 683 684 case TYPE_TRUE: 685 Py_INCREF(Py_True); 686 retval = Py_True; 687 break; 688 689 case TYPE_INT: 690 retval = PyInt_FromLong(r_long(p)); 691 break; 692 693 case TYPE_INT64: 694 retval = r_long64(p); 695 break; 696 697 case TYPE_LONG: 698 retval = r_PyLong(p); 699 break; 700 701 case TYPE_FLOAT: 702 { 703 char buf[256]; 704 double dx; 705 n = r_byte(p); 706 if (n == EOF || r_string(buf, (int)n, p) != n) { 707 PyErr_SetString(PyExc_EOFError, 708 "EOF read where object expected"); 709 retval = NULL; 710 break; 711 } 712 buf[n] = '\0'; 713 dx = PyOS_string_to_double(buf, NULL, NULL); 714 if (dx == -1.0 && PyErr_Occurred()) { 715 retval = NULL; 716 break; 717 } 718 retval = PyFloat_FromDouble(dx); 719 break; 720 } 721 722 case TYPE_BINARY_FLOAT: 723 { 724 unsigned char buf[8]; 725 double x; 726 if (r_string((char*)buf, 8, p) != 8) { 727 PyErr_SetString(PyExc_EOFError, 728 "EOF read where object expected"); 729 retval = NULL; 730 break; 731 } 732 x = _PyFloat_Unpack8(buf, 1); 733 if (x == -1.0 && PyErr_Occurred()) { 734 retval = NULL; 735 break; 736 } 737 retval = PyFloat_FromDouble(x); 738 break; 739 } 740 741 #ifndef WITHOUT_COMPLEX 742 case TYPE_COMPLEX: 743 { 744 char buf[256]; 745 Py_complex c; 746 n = r_byte(p); 747 if (n == EOF || r_string(buf, (int)n, p) != n) { 748 PyErr_SetString(PyExc_EOFError, 749 "EOF read where object expected"); 750 retval = NULL; 751 break; 752 } 753 buf[n] = '\0'; 754 c.real = PyOS_string_to_double(buf, NULL, NULL); 755 if (c.real == -1.0 && PyErr_Occurred()) { 756 retval = NULL; 757 break; 758 } 759 n = r_byte(p); 760 if (n == EOF || r_string(buf, (int)n, p) != n) { 761 PyErr_SetString(PyExc_EOFError, 762 "EOF read where object expected"); 763 retval = NULL; 764 break; 765 } 766 buf[n] = '\0'; 767 c.imag = PyOS_string_to_double(buf, NULL, NULL); 768 if (c.imag == -1.0 && PyErr_Occurred()) { 769 retval = NULL; 770 break; 771 } 772 retval = PyComplex_FromCComplex(c); 773 break; 774 } 775 776 case TYPE_BINARY_COMPLEX: 777 { 778 unsigned char buf[8]; 779 Py_complex c; 780 if (r_string((char*)buf, 8, p) != 8) { 781 PyErr_SetString(PyExc_EOFError, 782 "EOF read where object expected"); 783 retval = NULL; 784 break; 785 } 786 c.real = _PyFloat_Unpack8(buf, 1); 787 if (c.real == -1.0 && PyErr_Occurred()) { 788 retval = NULL; 789 break; 790 } 791 if (r_string((char*)buf, 8, p) != 8) { 792 PyErr_SetString(PyExc_EOFError, 793 "EOF read where object expected"); 794 retval = NULL; 795 break; 796 } 797 c.imag = _PyFloat_Unpack8(buf, 1); 798 if (c.imag == -1.0 && PyErr_Occurred()) { 799 retval = NULL; 800 break; 801 } 802 retval = PyComplex_FromCComplex(c); 803 break; 804 } 805 #endif 806 807 case TYPE_INTERNED: 808 case TYPE_STRING: 809 n = r_long(p); 810 if (n < 0 || n > INT_MAX) { 811 PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); 812 retval = NULL; 813 break; 814 } 815 v = PyString_FromStringAndSize((char *)NULL, n); 816 if (v == NULL) { 817 retval = NULL; 818 break; 819 } 820 if (r_string(PyString_AS_STRING(v), (int)n, p) != n) { 821 Py_DECREF(v); 822 PyErr_SetString(PyExc_EOFError, 823 "EOF read where object expected"); 824 retval = NULL; 825 break; 826 } 827 if (type == TYPE_INTERNED) { 828 PyString_InternInPlace(&v); 829 if (PyList_Append(p->strings, v) < 0) { 830 retval = NULL; 831 break; 832 } 833 } 834 retval = v; 835 break; 836 837 case TYPE_STRINGREF: 838 n = r_long(p); 839 if (n < 0 || n >= PyList_GET_SIZE(p->strings)) { 840 PyErr_SetString(PyExc_ValueError, "bad marshal data (string ref out of range)"); 841 retval = NULL; 842 break; 843 } 844 v = PyList_GET_ITEM(p->strings, n); 845 Py_INCREF(v); 846 retval = v; 847 break; 848 849 #ifdef Py_USING_UNICODE 850 case TYPE_UNICODE: 851 { 852 char *buffer; 853 854 n = r_long(p); 855 if (n < 0 || n > INT_MAX) { 856 PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)"); 857 retval = NULL; 858 break; 859 } 860 buffer = PyMem_NEW(char, n); 861 if (buffer == NULL) { 862 retval = PyErr_NoMemory(); 863 break; 864 } 865 if (r_string(buffer, (int)n, p) != n) { 866 PyMem_DEL(buffer); 867 PyErr_SetString(PyExc_EOFError, 868 "EOF read where object expected"); 869 retval = NULL; 870 break; 871 } 872 v = PyUnicode_DecodeUTF8(buffer, n, NULL); 873 PyMem_DEL(buffer); 874 retval = v; 875 break; 876 } 877 #endif 878 879 case TYPE_TUPLE: 880 n = r_long(p); 881 if (n < 0 || n > INT_MAX) { 882 PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)"); 883 retval = NULL; 884 break; 885 } 886 v = PyTuple_New((int)n); 887 if (v == NULL) { 888 retval = NULL; 889 break; 890 } 891 for (i = 0; i < n; i++) { 892 v2 = r_object(p); 893 if ( v2 == NULL ) { 894 if (!PyErr_Occurred()) 895 PyErr_SetString(PyExc_TypeError, 896 "NULL object in marshal data for tuple"); 897 Py_DECREF(v); 898 v = NULL; 899 break; 900 } 901 PyTuple_SET_ITEM(v, (int)i, v2); 902 } 903 retval = v; 904 break; 905 906 case TYPE_LIST: 907 n = r_long(p); 908 if (n < 0 || n > INT_MAX) { 909 PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)"); 910 retval = NULL; 911 break; 912 } 913 v = PyList_New((int)n); 914 if (v == NULL) { 915 retval = NULL; 916 break; 917 } 918 for (i = 0; i < n; i++) { 919 v2 = r_object(p); 920 if ( v2 == NULL ) { 921 if (!PyErr_Occurred()) 922 PyErr_SetString(PyExc_TypeError, 923 "NULL object in marshal data for list"); 924 Py_DECREF(v); 925 v = NULL; 926 break; 927 } 928 PyList_SET_ITEM(v, (int)i, v2); 929 } 930 retval = v; 931 break; 932 933 case TYPE_DICT: 934 v = PyDict_New(); 935 if (v == NULL) { 936 retval = NULL; 937 break; 938 } 939 for (;;) { 940 PyObject *key, *val; 941 key = r_object(p); 942 if (key == NULL) 943 break; 944 val = r_object(p); 945 if (val != NULL) 946 PyDict_SetItem(v, key, val); 947 Py_DECREF(key); 948 Py_XDECREF(val); 949 } 950 if (PyErr_Occurred()) { 951 Py_DECREF(v); 952 v = NULL; 953 } 954 retval = v; 955 break; 956 957 case TYPE_SET: 958 case TYPE_FROZENSET: 959 n = r_long(p); 960 if (n < 0 || n > INT_MAX) { 961 PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)"); 962 retval = NULL; 963 break; 964 } 965 v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL); 966 if (v == NULL) { 967 retval = NULL; 968 break; 969 } 970 for (i = 0; i < n; i++) { 971 v2 = r_object(p); 972 if ( v2 == NULL ) { 973 if (!PyErr_Occurred()) 974 PyErr_SetString(PyExc_TypeError, 975 "NULL object in marshal data for set"); 976 Py_DECREF(v); 977 v = NULL; 978 break; 979 } 980 if (PySet_Add(v, v2) == -1) { 981 Py_DECREF(v); 982 Py_DECREF(v2); 983 v = NULL; 984 break; 985 } 986 Py_DECREF(v2); 987 } 988 retval = v; 989 break; 990 991 case TYPE_CODE: 992 if (PyEval_GetRestricted()) { 993 PyErr_SetString(PyExc_RuntimeError, 994 "cannot unmarshal code objects in " 995 "restricted execution mode"); 996 retval = NULL; 997 break; 998 } 999 else { 1000 int argcount; 1001 int nlocals; 1002 int stacksize; 1003 int flags; 1004 PyObject *code = NULL; 1005 PyObject *consts = NULL; 1006 PyObject *names = NULL; 1007 PyObject *varnames = NULL; 1008 PyObject *freevars = NULL; 1009 PyObject *cellvars = NULL; 1010 PyObject *filename = NULL; 1011 PyObject *name = NULL; 1012 int firstlineno; 1013 PyObject *lnotab = NULL; 1014 1015 v = NULL; 1016 1017 /* XXX ignore long->int overflows for now */ 1018 argcount = (int)r_long(p); 1019 nlocals = (int)r_long(p); 1020 stacksize = (int)r_long(p); 1021 flags = (int)r_long(p); 1022 code = r_object(p); 1023 if (code == NULL) 1024 goto code_error; 1025 consts = r_object(p); 1026 if (consts == NULL) 1027 goto code_error; 1028 names = r_object(p); 1029 if (names == NULL) 1030 goto code_error; 1031 varnames = r_object(p); 1032 if (varnames == NULL) 1033 goto code_error; 1034 freevars = r_object(p); 1035 if (freevars == NULL) 1036 goto code_error; 1037 cellvars = r_object(p); 1038 if (cellvars == NULL) 1039 goto code_error; 1040 filename = r_object(p); 1041 if (filename == NULL) 1042 goto code_error; 1043 name = r_object(p); 1044 if (name == NULL) 1045 goto code_error; 1046 firstlineno = (int)r_long(p); 1047 lnotab = r_object(p); 1048 if (lnotab == NULL) 1049 goto code_error; 1050 1051 v = (PyObject *) PyCode_New( 1052 argcount, nlocals, stacksize, flags, 1053 code, consts, names, varnames, 1054 freevars, cellvars, filename, name, 1055 firstlineno, lnotab); 1056 1057 code_error: 1058 Py_XDECREF(code); 1059 Py_XDECREF(consts); 1060 Py_XDECREF(names); 1061 Py_XDECREF(varnames); 1062 Py_XDECREF(freevars); 1063 Py_XDECREF(cellvars); 1064 Py_XDECREF(filename); 1065 Py_XDECREF(name); 1066 Py_XDECREF(lnotab); 1067 1068 } 1069 retval = v; 1070 break; 1071 1072 default: 1073 /* Bogus data got written, which isn't ideal. 1074 This will let you keep working and recover. */ 1075 PyErr_SetString(PyExc_ValueError, "bad marshal data (unknown type code)"); 1076 retval = NULL; 1077 break; 1078 1079 } 1080 p->depth--; 1081 return retval; 1082 } 1083 1084 static PyObject * 1085 read_object(RFILE *p) 1086 { 1087 PyObject *v; 1088 if (PyErr_Occurred()) { 1089 fprintf(stderr, "XXX readobject called with exception set\n"); 1090 return NULL; 1091 } 1092 v = r_object(p); 1093 if (v == NULL && !PyErr_Occurred()) 1094 PyErr_SetString(PyExc_TypeError, "NULL object in marshal data for object"); 1095 return v; 1096 } 1097 1098 int 1099 PyMarshal_ReadShortFromFile(FILE *fp) 1100 { 1101 RFILE rf; 1102 assert(fp); 1103 rf.fp = fp; 1104 rf.strings = NULL; 1105 rf.end = rf.ptr = NULL; 1106 return r_short(&rf); 1107 } 1108 1109 long 1110 PyMarshal_ReadLongFromFile(FILE *fp) 1111 { 1112 RFILE rf; 1113 rf.fp = fp; 1114 rf.strings = NULL; 1115 rf.ptr = rf.end = NULL; 1116 return r_long(&rf); 1117 } 1118 1119 #ifdef HAVE_FSTAT 1120 /* Return size of file in bytes; < 0 if unknown. */ 1121 static off_t 1122 getfilesize(FILE *fp) 1123 { 1124 struct stat st; 1125 if (fstat(fileno(fp), &st) != 0) 1126 return -1; 1127 else 1128 return st.st_size; 1129 } 1130 #endif 1131 1132 /* If we can get the size of the file up-front, and it's reasonably small, 1133 * read it in one gulp and delegate to ...FromString() instead. Much quicker 1134 * than reading a byte at a time from file; speeds .pyc imports. 1135 * CAUTION: since this may read the entire remainder of the file, don't 1136 * call it unless you know you're done with the file. 1137 */ 1138 PyObject * 1139 PyMarshal_ReadLastObjectFromFile(FILE *fp) 1140 { 1141 /* REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc. */ 1142 #define REASONABLE_FILE_LIMIT (1L << 18) 1143 #ifdef HAVE_FSTAT 1144 off_t filesize; 1145 filesize = getfilesize(fp); 1146 if (filesize > 0 && filesize <= REASONABLE_FILE_LIMIT) { 1147 char* pBuf = (char *)PyMem_MALLOC(filesize); 1148 if (pBuf != NULL) { 1149 PyObject* v; 1150 size_t n; 1151 /* filesize must fit into an int, because it 1152 is smaller than REASONABLE_FILE_LIMIT */ 1153 n = fread(pBuf, 1, (int)filesize, fp); 1154 v = PyMarshal_ReadObjectFromString(pBuf, n); 1155 PyMem_FREE(pBuf); 1156 return v; 1157 } 1158 1159 } 1160 #endif 1161 /* We don't have fstat, or we do but the file is larger than 1162 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time. 1163 */ 1164 return PyMarshal_ReadObjectFromFile(fp); 1165 1166 #undef REASONABLE_FILE_LIMIT 1167 } 1168 1169 PyObject * 1170 PyMarshal_ReadObjectFromFile(FILE *fp) 1171 { 1172 RFILE rf; 1173 PyObject *result; 1174 rf.fp = fp; 1175 rf.strings = PyList_New(0); 1176 rf.depth = 0; 1177 rf.ptr = rf.end = NULL; 1178 result = r_object(&rf); 1179 Py_DECREF(rf.strings); 1180 return result; 1181 } 1182 1183 PyObject * 1184 PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len) 1185 { 1186 RFILE rf; 1187 PyObject *result; 1188 rf.fp = NULL; 1189 rf.ptr = str; 1190 rf.end = str + len; 1191 rf.strings = PyList_New(0); 1192 rf.depth = 0; 1193 result = r_object(&rf); 1194 Py_DECREF(rf.strings); 1195 return result; 1196 } 1197 1198 static void 1199 set_error(int error) 1200 { 1201 switch (error) { 1202 case WFERR_NOMEMORY: 1203 PyErr_NoMemory(); 1204 break; 1205 case WFERR_UNMARSHALLABLE: 1206 PyErr_SetString(PyExc_ValueError, "unmarshallable object"); 1207 break; 1208 case WFERR_NESTEDTOODEEP: 1209 default: 1210 PyErr_SetString(PyExc_ValueError, 1211 "object too deeply nested to marshal"); 1212 break; 1213 } 1214 } 1215 1216 PyObject * 1217 PyMarshal_WriteObjectToString(PyObject *x, int version) 1218 { 1219 WFILE wf; 1220 wf.fp = NULL; 1221 wf.str = PyString_FromStringAndSize((char *)NULL, 50); 1222 if (wf.str == NULL) 1223 return NULL; 1224 wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str); 1225 wf.end = wf.ptr + PyString_Size(wf.str); 1226 wf.error = WFERR_OK; 1227 wf.depth = 0; 1228 wf.version = version; 1229 wf.strings = (version > 0) ? PyDict_New() : NULL; 1230 w_object(x, &wf); 1231 Py_XDECREF(wf.strings); 1232 if (wf.str != NULL) { 1233 char *base = PyString_AS_STRING((PyStringObject *)wf.str); 1234 if (wf.ptr - base > PY_SSIZE_T_MAX) { 1235 Py_DECREF(wf.str); 1236 PyErr_SetString(PyExc_OverflowError, 1237 "too much marshall data for a string"); 1238 return NULL; 1239 } 1240 if (_PyString_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base))) 1241 return NULL; 1242 } 1243 if (wf.error != WFERR_OK) { 1244 Py_XDECREF(wf.str); 1245 set_error(wf.error); 1246 return NULL; 1247 } 1248 return wf.str; 1249 } 1250 1251 /* And an interface for Python programs... */ 1252 1253 static PyObject * 1254 marshal_dump(PyObject *self, PyObject *args) 1255 { 1256 WFILE wf; 1257 PyObject *x; 1258 PyObject *f; 1259 int version = Py_MARSHAL_VERSION; 1260 if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version)) 1261 return NULL; 1262 if (!PyFile_Check(f)) { 1263 PyErr_SetString(PyExc_TypeError, 1264 "marshal.dump() 2nd arg must be file"); 1265 return NULL; 1266 } 1267 wf.fp = PyFile_AsFile(f); 1268 wf.str = NULL; 1269 wf.ptr = wf.end = NULL; 1270 wf.error = WFERR_OK; 1271 wf.depth = 0; 1272 wf.strings = (version > 0) ? PyDict_New() : 0; 1273 wf.version = version; 1274 w_object(x, &wf); 1275 Py_XDECREF(wf.strings); 1276 if (wf.error != WFERR_OK) { 1277 set_error(wf.error); 1278 return NULL; 1279 } 1280 Py_INCREF(Py_None); 1281 return Py_None; 1282 } 1283 1284 PyDoc_STRVAR(dump_doc, 1285 "dump(value, file[, version])\n\ 1286 \n\ 1287 Write the value on the open file. The value must be a supported type.\n\ 1288 The file must be an open file object such as sys.stdout or returned by\n\ 1289 open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\ 1290 \n\ 1291 If the value has (or contains an object that has) an unsupported type, a\n\ 1292 ValueError exception is raised but garbage data will also be written\n\ 1293 to the file. The object will not be properly read back by load()\n\ 1294 \n\ 1295 New in version 2.4: The version argument indicates the data format that\n\ 1296 dump should use."); 1297 1298 static PyObject * 1299 marshal_load(PyObject *self, PyObject *f) 1300 { 1301 RFILE rf; 1302 PyObject *result; 1303 if (!PyFile_Check(f)) { 1304 PyErr_SetString(PyExc_TypeError, 1305 "marshal.load() arg must be file"); 1306 return NULL; 1307 } 1308 rf.fp = PyFile_AsFile(f); 1309 rf.strings = PyList_New(0); 1310 rf.depth = 0; 1311 result = read_object(&rf); 1312 Py_DECREF(rf.strings); 1313 return result; 1314 } 1315 1316 PyDoc_STRVAR(load_doc, 1317 "load(file)\n\ 1318 \n\ 1319 Read one value from the open file and return it. If no valid value is\n\ 1320 read (e.g. because the data has a different Python versions\n\ 1321 incompatible marshal format), raise EOFError, ValueError or TypeError.\n\ 1322 The file must be an open file object opened in binary mode ('rb' or\n\ 1323 'r+b').\n\ 1324 \n\ 1325 Note: If an object containing an unsupported type was marshalled with\n\ 1326 dump(), load() will substitute None for the unmarshallable type."); 1327 1328 1329 static PyObject * 1330 marshal_dumps(PyObject *self, PyObject *args) 1331 { 1332 PyObject *x; 1333 int version = Py_MARSHAL_VERSION; 1334 if (!PyArg_ParseTuple(args, "O|i:dumps", &x, &version)) 1335 return NULL; 1336 return PyMarshal_WriteObjectToString(x, version); 1337 } 1338 1339 PyDoc_STRVAR(dumps_doc, 1340 "dumps(value[, version])\n\ 1341 \n\ 1342 Return the string that would be written to a file by dump(value, file).\n\ 1343 The value must be a supported type. Raise a ValueError exception if\n\ 1344 value has (or contains an object that has) an unsupported type.\n\ 1345 \n\ 1346 New in version 2.4: The version argument indicates the data format that\n\ 1347 dumps should use."); 1348 1349 1350 static PyObject * 1351 marshal_loads(PyObject *self, PyObject *args) 1352 { 1353 RFILE rf; 1354 char *s; 1355 Py_ssize_t n; 1356 PyObject* result; 1357 if (!PyArg_ParseTuple(args, "s#:loads", &s, &n)) 1358 return NULL; 1359 rf.fp = NULL; 1360 rf.ptr = s; 1361 rf.end = s + n; 1362 rf.strings = PyList_New(0); 1363 rf.depth = 0; 1364 result = read_object(&rf); 1365 Py_DECREF(rf.strings); 1366 return result; 1367 } 1368 1369 PyDoc_STRVAR(loads_doc, 1370 "loads(string)\n\ 1371 \n\ 1372 Convert the string to a value. If no valid value is found, raise\n\ 1373 EOFError, ValueError or TypeError. Extra characters in the string are\n\ 1374 ignored."); 1375 1376 static PyMethodDef marshal_methods[] = { 1377 {"dump", marshal_dump, METH_VARARGS, dump_doc}, 1378 {"load", marshal_load, METH_O, load_doc}, 1379 {"dumps", marshal_dumps, METH_VARARGS, dumps_doc}, 1380 {"loads", marshal_loads, METH_VARARGS, loads_doc}, 1381 {NULL, NULL} /* sentinel */ 1382 }; 1383 1384 PyDoc_STRVAR(marshal_doc, 1385 "This module contains functions that can read and write Python values in\n\ 1386 a binary format. The format is specific to Python, but independent of\n\ 1387 machine architecture issues.\n\ 1388 \n\ 1389 Not all Python object types are supported; in general, only objects\n\ 1390 whose value is independent from a particular invocation of Python can be\n\ 1391 written and read by this module. The following types are supported:\n\ 1392 None, integers, long integers, floating point numbers, strings, Unicode\n\ 1393 objects, tuples, lists, sets, dictionaries, and code objects, where it\n\ 1394 should be understood that tuples, lists and dictionaries are only\n\ 1395 supported as long as the values contained therein are themselves\n\ 1396 supported; and recursive lists and dictionaries should not be written\n\ 1397 (they will cause infinite loops).\n\ 1398 \n\ 1399 Variables:\n\ 1400 \n\ 1401 version -- indicates the format that the module uses. Version 0 is the\n\ 1402 historical format, version 1 (added in Python 2.4) shares interned\n\ 1403 strings and version 2 (added in Python 2.5) uses a binary format for\n\ 1404 floating point numbers. (New in version 2.4)\n\ 1405 \n\ 1406 Functions:\n\ 1407 \n\ 1408 dump() -- write value to a file\n\ 1409 load() -- read value from a file\n\ 1410 dumps() -- write value to a string\n\ 1411 loads() -- read value from a string"); 1412 1413 1414 PyMODINIT_FUNC 1415 PyMarshal_Init(void) 1416 { 1417 PyObject *mod = Py_InitModule3("marshal", marshal_methods, 1418 marshal_doc); 1419 if (mod == NULL) 1420 return; 1421 PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION); 1422 } 1423
