1 # -*- coding: utf-8 -*- 2 import unittest 3 import pickle 4 import cPickle 5 import StringIO 6 import cStringIO 7 import pickletools 8 import copy_reg 9 import sys 10 11 from test import test_support as support 12 from test.test_support import TestFailed, verbose, have_unicode, TESTFN 13 try: 14 from test.test_support import _2G, _1M, precisionbigmemtest 15 except ImportError: 16 # this import might fail when run on older Python versions by test_xpickle 17 _2G = _1M = 0 18 def precisionbigmemtest(*args, **kwargs): 19 return lambda self: None 20 21 # Tests that try a number of pickle protocols should have a 22 # for proto in protocols: 23 # kind of outer loop. 24 assert pickle.HIGHEST_PROTOCOL == cPickle.HIGHEST_PROTOCOL == 2 25 protocols = range(pickle.HIGHEST_PROTOCOL + 1) 26 27 # Copy of test.test_support.run_with_locale. This is needed to support Python 28 # 2.4, which didn't include it. This is all to support test_xpickle, which 29 # bounces pickled objects through older Python versions to test backwards 30 # compatibility. 31 def run_with_locale(catstr, *locales): 32 def decorator(func): 33 def inner(*args, **kwds): 34 try: 35 import locale 36 category = getattr(locale, catstr) 37 orig_locale = locale.setlocale(category) 38 except AttributeError: 39 # if the test author gives us an invalid category string 40 raise 41 except: 42 # cannot retrieve original locale, so do nothing 43 locale = orig_locale = None 44 else: 45 for loc in locales: 46 try: 47 locale.setlocale(category, loc) 48 break 49 except: 50 pass 51 52 # now run the function, resetting the locale on exceptions 53 try: 54 return func(*args, **kwds) 55 finally: 56 if locale and orig_locale: 57 locale.setlocale(category, orig_locale) 58 inner.func_name = func.func_name 59 inner.__doc__ = func.__doc__ 60 return inner 61 return decorator 62 63 def no_tracing(func): 64 """Decorator to temporarily turn off tracing for the duration of a test.""" 65 if not hasattr(sys, 'gettrace'): 66 return func 67 else: 68 def wrapper(*args, **kwargs): 69 original_trace = sys.gettrace() 70 try: 71 sys.settrace(None) 72 return func(*args, **kwargs) 73 finally: 74 sys.settrace(original_trace) 75 wrapper.__name__ = func.__name__ 76 return wrapper 77 78 79 # Return True if opcode code appears in the pickle, else False. 80 def opcode_in_pickle(code, pickle): 81 for op, dummy, dummy in pickletools.genops(pickle): 82 if op.code == code: 83 return True 84 return False 85 86 # Return the number of times opcode code appears in pickle. 87 def count_opcode(code, pickle): 88 n = 0 89 for op, dummy, dummy in pickletools.genops(pickle): 90 if op.code == code: 91 n += 1 92 return n 93 94 class UnseekableIO(StringIO.StringIO): 95 def peek(self, *args): 96 raise NotImplementedError 97 98 def seek(self, *args): 99 raise NotImplementedError 100 101 def tell(self): 102 raise NotImplementedError 103 104 # We can't very well test the extension registry without putting known stuff 105 # in it, but we have to be careful to restore its original state. Code 106 # should do this: 107 # 108 # e = ExtensionSaver(extension_code) 109 # try: 110 # fiddle w/ the extension registry's stuff for extension_code 111 # finally: 112 # e.restore() 113 114 class ExtensionSaver: 115 # Remember current registration for code (if any), and remove it (if 116 # there is one). 117 def __init__(self, code): 118 self.code = code 119 if code in copy_reg._inverted_registry: 120 self.pair = copy_reg._inverted_registry[code] 121 copy_reg.remove_extension(self.pair[0], self.pair[1], code) 122 else: 123 self.pair = None 124 125 # Restore previous registration for code. 126 def restore(self): 127 code = self.code 128 curpair = copy_reg._inverted_registry.get(code) 129 if curpair is not None: 130 copy_reg.remove_extension(curpair[0], curpair[1], code) 131 pair = self.pair 132 if pair is not None: 133 copy_reg.add_extension(pair[0], pair[1], code) 134 135 class C: 136 def __cmp__(self, other): 137 return cmp(self.__dict__, other.__dict__) 138 139 class D(C): 140 def __init__(self, arg): 141 pass 142 143 class E(C): 144 def __getinitargs__(self): 145 return () 146 147 class H(object): 148 pass 149 150 # Hashable mutable key 151 class K(object): 152 def __init__(self, value): 153 self.value = value 154 155 def __reduce__(self): 156 # Shouldn't support the recursion itself 157 return K, (self.value,) 158 159 import __main__ 160 __main__.C = C 161 C.__module__ = "__main__" 162 __main__.D = D 163 D.__module__ = "__main__" 164 __main__.E = E 165 E.__module__ = "__main__" 166 __main__.H = H 167 H.__module__ = "__main__" 168 __main__.K = K 169 K.__module__ = "__main__" 170 171 class myint(int): 172 def __init__(self, x): 173 self.str = str(x) 174 175 class initarg(C): 176 177 def __init__(self, a, b): 178 self.a = a 179 self.b = b 180 181 def __getinitargs__(self): 182 return self.a, self.b 183 184 class metaclass(type): 185 pass 186 187 class use_metaclass(object): 188 __metaclass__ = metaclass 189 190 class pickling_metaclass(type): 191 def __eq__(self, other): 192 return (type(self) == type(other) and 193 self.reduce_args == other.reduce_args) 194 195 def __reduce__(self): 196 return (create_dynamic_class, self.reduce_args) 197 198 __hash__ = None 199 200 def create_dynamic_class(name, bases): 201 result = pickling_metaclass(name, bases, dict()) 202 result.reduce_args = (name, bases) 203 return result 204 205 # DATA0 .. DATA2 are the pickles we expect under the various protocols, for 206 # the object returned by create_data(). 207 208 # break into multiple strings to avoid confusing font-lock-mode 209 DATA0 = """(lp1 210 I0 211 aL1L 212 aF2 213 ac__builtin__ 214 complex 215 p2 216 """ + \ 217 """(F3 218 F0 219 tRp3 220 aI1 221 aI-1 222 aI255 223 aI-255 224 aI-256 225 aI65535 226 aI-65535 227 aI-65536 228 aI2147483647 229 aI-2147483647 230 aI-2147483648 231 a""" + \ 232 """(S'abc' 233 p4 234 g4 235 """ + \ 236 """(i__main__ 237 C 238 p5 239 """ + \ 240 """(dp6 241 S'foo' 242 p7 243 I1 244 sS'bar' 245 p8 246 I2 247 sbg5 248 tp9 249 ag9 250 aI5 251 a. 252 """ 253 254 # Disassembly of DATA0. 255 DATA0_DIS = """\ 256 0: ( MARK 257 1: l LIST (MARK at 0) 258 2: p PUT 1 259 5: I INT 0 260 8: a APPEND 261 9: L LONG 1L 262 13: a APPEND 263 14: F FLOAT 2.0 264 17: a APPEND 265 18: c GLOBAL '__builtin__ complex' 266 39: p PUT 2 267 42: ( MARK 268 43: F FLOAT 3.0 269 46: F FLOAT 0.0 270 49: t TUPLE (MARK at 42) 271 50: R REDUCE 272 51: p PUT 3 273 54: a APPEND 274 55: I INT 1 275 58: a APPEND 276 59: I INT -1 277 63: a APPEND 278 64: I INT 255 279 69: a APPEND 280 70: I INT -255 281 76: a APPEND 282 77: I INT -256 283 83: a APPEND 284 84: I INT 65535 285 91: a APPEND 286 92: I INT -65535 287 100: a APPEND 288 101: I INT -65536 289 109: a APPEND 290 110: I INT 2147483647 291 122: a APPEND 292 123: I INT -2147483647 293 136: a APPEND 294 137: I INT -2147483648 295 150: a APPEND 296 151: ( MARK 297 152: S STRING 'abc' 298 159: p PUT 4 299 162: g GET 4 300 165: ( MARK 301 166: i INST '__main__ C' (MARK at 165) 302 178: p PUT 5 303 181: ( MARK 304 182: d DICT (MARK at 181) 305 183: p PUT 6 306 186: S STRING 'foo' 307 193: p PUT 7 308 196: I INT 1 309 199: s SETITEM 310 200: S STRING 'bar' 311 207: p PUT 8 312 210: I INT 2 313 213: s SETITEM 314 214: b BUILD 315 215: g GET 5 316 218: t TUPLE (MARK at 151) 317 219: p PUT 9 318 222: a APPEND 319 223: g GET 9 320 226: a APPEND 321 227: I INT 5 322 230: a APPEND 323 231: . STOP 324 highest protocol among opcodes = 0 325 """ 326 327 DATA1 = (']q\x01(K\x00L1L\nG@\x00\x00\x00\x00\x00\x00\x00' 328 'c__builtin__\ncomplex\nq\x02(G@\x08\x00\x00\x00\x00\x00' 329 '\x00G\x00\x00\x00\x00\x00\x00\x00\x00tRq\x03K\x01J\xff\xff' 330 '\xff\xffK\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xff' 331 'J\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00' 332 '\x00\x80J\x00\x00\x00\x80(U\x03abcq\x04h\x04(c__main__\n' 333 'C\nq\x05oq\x06}q\x07(U\x03fooq\x08K\x01U\x03barq\tK\x02ubh' 334 '\x06tq\nh\nK\x05e.' 335 ) 336 337 # Disassembly of DATA1. 338 DATA1_DIS = """\ 339 0: ] EMPTY_LIST 340 1: q BINPUT 1 341 3: ( MARK 342 4: K BININT1 0 343 6: L LONG 1L 344 10: G BINFLOAT 2.0 345 19: c GLOBAL '__builtin__ complex' 346 40: q BINPUT 2 347 42: ( MARK 348 43: G BINFLOAT 3.0 349 52: G BINFLOAT 0.0 350 61: t TUPLE (MARK at 42) 351 62: R REDUCE 352 63: q BINPUT 3 353 65: K BININT1 1 354 67: J BININT -1 355 72: K BININT1 255 356 74: J BININT -255 357 79: J BININT -256 358 84: M BININT2 65535 359 87: J BININT -65535 360 92: J BININT -65536 361 97: J BININT 2147483647 362 102: J BININT -2147483647 363 107: J BININT -2147483648 364 112: ( MARK 365 113: U SHORT_BINSTRING 'abc' 366 118: q BINPUT 4 367 120: h BINGET 4 368 122: ( MARK 369 123: c GLOBAL '__main__ C' 370 135: q BINPUT 5 371 137: o OBJ (MARK at 122) 372 138: q BINPUT 6 373 140: } EMPTY_DICT 374 141: q BINPUT 7 375 143: ( MARK 376 144: U SHORT_BINSTRING 'foo' 377 149: q BINPUT 8 378 151: K BININT1 1 379 153: U SHORT_BINSTRING 'bar' 380 158: q BINPUT 9 381 160: K BININT1 2 382 162: u SETITEMS (MARK at 143) 383 163: b BUILD 384 164: h BINGET 6 385 166: t TUPLE (MARK at 112) 386 167: q BINPUT 10 387 169: h BINGET 10 388 171: K BININT1 5 389 173: e APPENDS (MARK at 3) 390 174: . STOP 391 highest protocol among opcodes = 1 392 """ 393 394 DATA2 = ('\x80\x02]q\x01(K\x00\x8a\x01\x01G@\x00\x00\x00\x00\x00\x00\x00' 395 'c__builtin__\ncomplex\nq\x02G@\x08\x00\x00\x00\x00\x00\x00G\x00' 396 '\x00\x00\x00\x00\x00\x00\x00\x86Rq\x03K\x01J\xff\xff\xff\xffK' 397 '\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xff' 398 'J\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00' 399 '\x80(U\x03abcq\x04h\x04(c__main__\nC\nq\x05oq\x06}q\x07(U\x03foo' 400 'q\x08K\x01U\x03barq\tK\x02ubh\x06tq\nh\nK\x05e.') 401 402 # Disassembly of DATA2. 403 DATA2_DIS = """\ 404 0: \x80 PROTO 2 405 2: ] EMPTY_LIST 406 3: q BINPUT 1 407 5: ( MARK 408 6: K BININT1 0 409 8: \x8a LONG1 1L 410 11: G BINFLOAT 2.0 411 20: c GLOBAL '__builtin__ complex' 412 41: q BINPUT 2 413 43: G BINFLOAT 3.0 414 52: G BINFLOAT 0.0 415 61: \x86 TUPLE2 416 62: R REDUCE 417 63: q BINPUT 3 418 65: K BININT1 1 419 67: J BININT -1 420 72: K BININT1 255 421 74: J BININT -255 422 79: J BININT -256 423 84: M BININT2 65535 424 87: J BININT -65535 425 92: J BININT -65536 426 97: J BININT 2147483647 427 102: J BININT -2147483647 428 107: J BININT -2147483648 429 112: ( MARK 430 113: U SHORT_BINSTRING 'abc' 431 118: q BINPUT 4 432 120: h BINGET 4 433 122: ( MARK 434 123: c GLOBAL '__main__ C' 435 135: q BINPUT 5 436 137: o OBJ (MARK at 122) 437 138: q BINPUT 6 438 140: } EMPTY_DICT 439 141: q BINPUT 7 440 143: ( MARK 441 144: U SHORT_BINSTRING 'foo' 442 149: q BINPUT 8 443 151: K BININT1 1 444 153: U SHORT_BINSTRING 'bar' 445 158: q BINPUT 9 446 160: K BININT1 2 447 162: u SETITEMS (MARK at 143) 448 163: b BUILD 449 164: h BINGET 6 450 166: t TUPLE (MARK at 112) 451 167: q BINPUT 10 452 169: h BINGET 10 453 171: K BININT1 5 454 173: e APPENDS (MARK at 5) 455 174: . STOP 456 highest protocol among opcodes = 2 457 """ 458 459 def create_data(): 460 c = C() 461 c.foo = 1 462 c.bar = 2 463 x = [0, 1L, 2.0, 3.0+0j] 464 # Append some integer test cases at cPickle.c's internal size 465 # cutoffs. 466 uint1max = 0xff 467 uint2max = 0xffff 468 int4max = 0x7fffffff 469 x.extend([1, -1, 470 uint1max, -uint1max, -uint1max-1, 471 uint2max, -uint2max, -uint2max-1, 472 int4max, -int4max, -int4max-1]) 473 y = ('abc', 'abc', c, c) 474 x.append(y) 475 x.append(y) 476 x.append(5) 477 return x 478 479 480 class AbstractUnpickleTests(unittest.TestCase): 481 # Subclass must define self.loads, self.error. 482 483 _testdata = create_data() 484 485 def assert_is_copy(self, obj, objcopy, msg=None): 486 """Utility method to verify if two objects are copies of each others. 487 """ 488 if msg is None: 489 msg = "{!r} is not a copy of {!r}".format(obj, objcopy) 490 self.assertEqual(obj, objcopy, msg=msg) 491 self.assertIs(type(obj), type(objcopy), msg=msg) 492 if hasattr(obj, '__dict__'): 493 self.assertDictEqual(obj.__dict__, objcopy.__dict__, msg=msg) 494 self.assertIsNot(obj.__dict__, objcopy.__dict__, msg=msg) 495 if hasattr(obj, '__slots__'): 496 self.assertListEqual(obj.__slots__, objcopy.__slots__, msg=msg) 497 for slot in obj.__slots__: 498 self.assertEqual( 499 hasattr(obj, slot), hasattr(objcopy, slot), msg=msg) 500 self.assertEqual(getattr(obj, slot, None), 501 getattr(objcopy, slot, None), msg=msg) 502 503 def check_unpickling_error(self, errors, data): 504 try: 505 try: 506 self.loads(data) 507 except: 508 if support.verbose > 1: 509 exc_type, exc, tb = sys.exc_info() 510 print '%-32r - %s: %s' % (data, exc_type.__name__, exc) 511 raise 512 except errors: 513 pass 514 else: 515 try: 516 exc_name = errors.__name__ 517 except AttributeError: 518 exc_name = str(errors) 519 raise self.failureException('%s not raised' % exc_name) 520 521 def test_load_from_canned_string(self): 522 expected = self._testdata 523 for canned in DATA0, DATA1, DATA2: 524 got = self.loads(canned) 525 self.assert_is_copy(expected, got) 526 527 def test_garyp(self): 528 self.check_unpickling_error(self.error, 'garyp') 529 530 def test_maxint64(self): 531 maxint64 = (1L << 63) - 1 532 data = 'I' + str(maxint64) + '\n.' 533 got = self.loads(data) 534 self.assertEqual(got, maxint64) 535 536 # Try too with a bogus literal. 537 data = 'I' + str(maxint64) + 'JUNK\n.' 538 self.check_unpickling_error(ValueError, data) 539 540 def test_insecure_strings(self): 541 insecure = ["abc", "2 + 2", # not quoted 542 #"'abc' + 'def'", # not a single quoted string 543 "'abc", # quote is not closed 544 "'abc\"", # open quote and close quote don't match 545 "'abc' ?", # junk after close quote 546 "'\\'", # trailing backslash 547 # issue #17710 548 "'", '"', 549 "' ", '" ', 550 '\'"', '"\'', 551 " ''", ' ""', 552 ' ', 553 # some tests of the quoting rules 554 #"'abc\"\''", 555 #"'\\\\a\'\'\'\\\'\\\\\''", 556 ] 557 for s in insecure: 558 buf = "S" + s + "\n." 559 self.check_unpickling_error(ValueError, buf) 560 561 def test_correctly_quoted_string(self): 562 goodpickles = [("S''\n.", ''), 563 ('S""\n.', ''), 564 ('S"\\n"\n.', '\n'), 565 ("S'\\n'\n.", '\n')] 566 for p, expected in goodpickles: 567 self.assertEqual(self.loads(p), expected) 568 569 def test_load_classic_instance(self): 570 # See issue5180. Test loading 2.x pickles that 571 # contain an instance of old style class. 572 for X, args in [(C, ()), (D, ('x',)), (E, ())]: 573 xname = X.__name__.encode('ascii') 574 # Protocol 0 (text mode pickle): 575 """ 576 0: ( MARK 577 1: i INST '__main__ X' (MARK at 0) 578 13: p PUT 0 579 16: ( MARK 580 17: d DICT (MARK at 16) 581 18: p PUT 1 582 21: b BUILD 583 22: . STOP 584 """ 585 pickle0 = ("(i__main__\n" 586 "X\n" 587 "p0\n" 588 "(dp1\nb.").replace('X', xname) 589 self.assert_is_copy(X(*args), self.loads(pickle0)) 590 591 # Protocol 1 (binary mode pickle) 592 """ 593 0: ( MARK 594 1: c GLOBAL '__main__ X' 595 13: q BINPUT 0 596 15: o OBJ (MARK at 0) 597 16: q BINPUT 1 598 18: } EMPTY_DICT 599 19: q BINPUT 2 600 21: b BUILD 601 22: . STOP 602 """ 603 pickle1 = ('(c__main__\n' 604 'X\n' 605 'q\x00oq\x01}q\x02b.').replace('X', xname) 606 self.assert_is_copy(X(*args), self.loads(pickle1)) 607 608 # Protocol 2 (pickle2 = '\x80\x02' + pickle1) 609 """ 610 0: \x80 PROTO 2 611 2: ( MARK 612 3: c GLOBAL '__main__ X' 613 15: q BINPUT 0 614 17: o OBJ (MARK at 2) 615 18: q BINPUT 1 616 20: } EMPTY_DICT 617 21: q BINPUT 2 618 23: b BUILD 619 24: . STOP 620 """ 621 pickle2 = ('\x80\x02(c__main__\n' 622 'X\n' 623 'q\x00oq\x01}q\x02b.').replace('X', xname) 624 self.assert_is_copy(X(*args), self.loads(pickle2)) 625 626 def test_load_str(self): 627 # From Python 2: pickle.dumps('a\x00\xa0', protocol=0) 628 self.assertEqual(self.loads("S'a\\x00\\xa0'\n."), 'a\x00\xa0') 629 # From Python 2: pickle.dumps('a\x00\xa0', protocol=1) 630 self.assertEqual(self.loads('U\x03a\x00\xa0.'), 'a\x00\xa0') 631 # From Python 2: pickle.dumps('a\x00\xa0', protocol=2) 632 self.assertEqual(self.loads('\x80\x02U\x03a\x00\xa0.'), 'a\x00\xa0') 633 634 def test_load_unicode(self): 635 # From Python 2: pickle.dumps(u'', protocol=0) 636 self.assertEqual(self.loads('V\\u03c0\n.'), u'') 637 # From Python 2: pickle.dumps(u'', protocol=1) 638 self.assertEqual(self.loads('X\x02\x00\x00\x00\xcf\x80.'), u'') 639 # From Python 2: pickle.dumps(u'', protocol=2) 640 self.assertEqual(self.loads('\x80\x02X\x02\x00\x00\x00\xcf\x80.'), u'') 641 642 def test_constants(self): 643 self.assertIsNone(self.loads('N.')) 644 self.assertIs(self.loads('\x88.'), True) 645 self.assertIs(self.loads('\x89.'), False) 646 self.assertIs(self.loads('I01\n.'), True) 647 self.assertIs(self.loads('I00\n.'), False) 648 649 def test_misc_get(self): 650 self.check_unpickling_error(self.error, 'g0\np0\n') 651 self.check_unpickling_error(self.error, 'h\x00q\x00') 652 653 def test_get(self): 654 pickled = '((lp100000\ng100000\nt.' 655 unpickled = self.loads(pickled) 656 self.assertEqual(unpickled, ([],)*2) 657 self.assertIs(unpickled[0], unpickled[1]) 658 659 def test_binget(self): 660 pickled = '(]q\xffh\xfft.' 661 unpickled = self.loads(pickled) 662 self.assertEqual(unpickled, ([],)*2) 663 self.assertIs(unpickled[0], unpickled[1]) 664 665 def test_long_binget(self): 666 pickled = '(]r\x00\x00\x01\x00j\x00\x00\x01\x00t.' 667 unpickled = self.loads(pickled) 668 self.assertEqual(unpickled, ([],)*2) 669 self.assertIs(unpickled[0], unpickled[1]) 670 671 def test_dup(self): 672 pickled = '((l2t.' 673 unpickled = self.loads(pickled) 674 self.assertEqual(unpickled, ([],)*2) 675 self.assertIs(unpickled[0], unpickled[1]) 676 677 def test_bad_stack(self): 678 badpickles = [ 679 '.', # STOP 680 '0', # POP 681 '1', # POP_MARK 682 '2', # DUP 683 # '(2', # PyUnpickler doesn't raise 684 'R', # REDUCE 685 ')R', 686 'a', # APPEND 687 'Na', 688 'b', # BUILD 689 'Nb', 690 'd', # DICT 691 'e', # APPENDS 692 # '(e', # PyUnpickler raises AttributeError 693 'i__builtin__\nlist\n', # INST 694 'l', # LIST 695 'o', # OBJ 696 '(o', 697 'p1\n', # PUT 698 'q\x00', # BINPUT 699 'r\x00\x00\x00\x00', # LONG_BINPUT 700 's', # SETITEM 701 'Ns', 702 'NNs', 703 't', # TUPLE 704 'u', # SETITEMS 705 # '(u', # PyUnpickler doesn't raise 706 '}(Nu', 707 '\x81', # NEWOBJ 708 ')\x81', 709 '\x85', # TUPLE1 710 '\x86', # TUPLE2 711 'N\x86', 712 '\x87', # TUPLE3 713 'N\x87', 714 'NN\x87', 715 ] 716 for p in badpickles: 717 self.check_unpickling_error(self.bad_stack_errors, p) 718 719 def test_bad_mark(self): 720 badpickles = [ 721 # 'N(.', # STOP 722 'N(2', # DUP 723 'c__builtin__\nlist\n)(R', # REDUCE 724 'c__builtin__\nlist\n()R', 725 ']N(a', # APPEND 726 # BUILD 727 'c__builtin__\nValueError\n)R}(b', 728 'c__builtin__\nValueError\n)R(}b', 729 '(Nd', # DICT 730 'N(p1\n', # PUT 731 'N(q\x00', # BINPUT 732 'N(r\x00\x00\x00\x00', # LONG_BINPUT 733 '}NN(s', # SETITEM 734 '}N(Ns', 735 '}(NNs', 736 '}((u', # SETITEMS 737 # NEWOBJ 738 'c__builtin__\nlist\n)(\x81', 739 'c__builtin__\nlist\n()\x81', 740 'N(\x85', # TUPLE1 741 'NN(\x86', # TUPLE2 742 'N(N\x86', 743 'NNN(\x87', # TUPLE3 744 'NN(N\x87', 745 'N(NN\x87', 746 ] 747 for p in badpickles: 748 self.check_unpickling_error(self.bad_mark_errors, p) 749 750 def test_truncated_data(self): 751 self.check_unpickling_error(EOFError, '') 752 self.check_unpickling_error(EOFError, 'N') 753 badpickles = [ 754 'F', # FLOAT 755 'F0.0', 756 'F0.00', 757 'G', # BINFLOAT 758 'G\x00\x00\x00\x00\x00\x00\x00', 759 'I', # INT 760 'I0', 761 'J', # BININT 762 'J\x00\x00\x00', 763 'K', # BININT1 764 'L', # LONG 765 'L0', 766 'L10', 767 'L0L', 768 'L10L', 769 'M', # BININT2 770 'M\x00', 771 # 'P', # PERSID 772 # 'Pabc', 773 'S', # STRING 774 "S'abc'", 775 'T', # BINSTRING 776 'T\x03\x00\x00', 777 'T\x03\x00\x00\x00', 778 'T\x03\x00\x00\x00ab', 779 'U', # SHORT_BINSTRING 780 'U\x03', 781 'U\x03ab', 782 'V', # UNICODE 783 'Vabc', 784 'X', # BINUNICODE 785 'X\x03\x00\x00', 786 'X\x03\x00\x00\x00', 787 'X\x03\x00\x00\x00ab', 788 '(c', # GLOBAL 789 '(c__builtin__', 790 '(c__builtin__\n', 791 '(c__builtin__\nlist', 792 'Ng', # GET 793 'Ng0', 794 '(i', # INST 795 '(i__builtin__', 796 '(i__builtin__\n', 797 '(i__builtin__\nlist', 798 'Nh', # BINGET 799 'Nj', # LONG_BINGET 800 'Nj\x00\x00\x00', 801 'Np', # PUT 802 'Np0', 803 'Nq', # BINPUT 804 'Nr', # LONG_BINPUT 805 'Nr\x00\x00\x00', 806 '\x80', # PROTO 807 '\x82', # EXT1 808 '\x83', # EXT2 809 '\x84\x01', 810 '\x84', # EXT4 811 '\x84\x01\x00\x00', 812 '\x8a', # LONG1 813 '\x8b', # LONG4 814 '\x8b\x00\x00\x00', 815 ] 816 for p in badpickles: 817 self.check_unpickling_error(self.truncated_errors, p) 818 819 820 class AbstractPickleTests(unittest.TestCase): 821 # Subclass must define self.dumps, self.loads. 822 823 _testdata = AbstractUnpickleTests._testdata 824 825 def setUp(self): 826 pass 827 828 def test_misc(self): 829 # test various datatypes not tested by testdata 830 for proto in protocols: 831 x = myint(4) 832 s = self.dumps(x, proto) 833 y = self.loads(s) 834 self.assertEqual(x, y) 835 836 x = (1, ()) 837 s = self.dumps(x, proto) 838 y = self.loads(s) 839 self.assertEqual(x, y) 840 841 x = initarg(1, x) 842 s = self.dumps(x, proto) 843 y = self.loads(s) 844 self.assertEqual(x, y) 845 846 # XXX test __reduce__ protocol? 847 848 def test_roundtrip_equality(self): 849 expected = self._testdata 850 for proto in protocols: 851 s = self.dumps(expected, proto) 852 got = self.loads(s) 853 self.assertEqual(expected, got) 854 855 # There are gratuitous differences between pickles produced by 856 # pickle and cPickle, largely because cPickle starts PUT indices at 857 # 1 and pickle starts them at 0. See XXX comment in cPickle's put2() -- 858 # there's a comment with an exclamation point there whose meaning 859 # is a mystery. cPickle also suppresses PUT for objects with a refcount 860 # of 1. 861 def dont_test_disassembly(self): 862 from pickletools import dis 863 864 for proto, expected in (0, DATA0_DIS), (1, DATA1_DIS): 865 s = self.dumps(self._testdata, proto) 866 filelike = cStringIO.StringIO() 867 dis(s, out=filelike) 868 got = filelike.getvalue() 869 self.assertEqual(expected, got) 870 871 def test_recursive_list(self): 872 l = [] 873 l.append(l) 874 for proto in protocols: 875 s = self.dumps(l, proto) 876 x = self.loads(s) 877 self.assertIsInstance(x, list) 878 self.assertEqual(len(x), 1) 879 self.assertIs(x[0], x) 880 881 def test_recursive_tuple_and_list(self): 882 t = ([],) 883 t[0].append(t) 884 for proto in protocols: 885 s = self.dumps(t, proto) 886 x = self.loads(s) 887 self.assertIsInstance(x, tuple) 888 self.assertEqual(len(x), 1) 889 self.assertIsInstance(x[0], list) 890 self.assertEqual(len(x[0]), 1) 891 self.assertIs(x[0][0], x) 892 893 def test_recursive_dict(self): 894 d = {} 895 d[1] = d 896 for proto in protocols: 897 s = self.dumps(d, proto) 898 x = self.loads(s) 899 self.assertIsInstance(x, dict) 900 self.assertEqual(x.keys(), [1]) 901 self.assertIs(x[1], x) 902 903 def test_recursive_dict_key(self): 904 d = {} 905 k = K(d) 906 d[k] = 1 907 for proto in protocols: 908 s = self.dumps(d, proto) 909 x = self.loads(s) 910 self.assertIsInstance(x, dict) 911 self.assertEqual(len(x.keys()), 1) 912 self.assertIsInstance(x.keys()[0], K) 913 self.assertIs(x.keys()[0].value, x) 914 915 def test_recursive_list_subclass(self): 916 y = MyList() 917 y.append(y) 918 s = self.dumps(y, 2) 919 x = self.loads(s) 920 self.assertIsInstance(x, MyList) 921 self.assertEqual(len(x), 1) 922 self.assertIs(x[0], x) 923 924 def test_recursive_dict_subclass(self): 925 d = MyDict() 926 d[1] = d 927 s = self.dumps(d, 2) 928 x = self.loads(s) 929 self.assertIsInstance(x, MyDict) 930 self.assertEqual(x.keys(), [1]) 931 self.assertIs(x[1], x) 932 933 def test_recursive_dict_subclass_key(self): 934 d = MyDict() 935 k = K(d) 936 d[k] = 1 937 s = self.dumps(d, 2) 938 x = self.loads(s) 939 self.assertIsInstance(x, MyDict) 940 self.assertEqual(len(x.keys()), 1) 941 self.assertIsInstance(x.keys()[0], K) 942 self.assertIs(x.keys()[0].value, x) 943 944 def test_recursive_inst(self): 945 i = C() 946 i.attr = i 947 for proto in protocols: 948 s = self.dumps(i, proto) 949 x = self.loads(s) 950 self.assertIsInstance(x, C) 951 self.assertEqual(dir(x), dir(i)) 952 self.assertIs(x.attr, x) 953 954 def test_recursive_multi(self): 955 l = [] 956 d = {1:l} 957 i = C() 958 i.attr = d 959 l.append(i) 960 for proto in protocols: 961 s = self.dumps(l, proto) 962 x = self.loads(s) 963 self.assertIsInstance(x, list) 964 self.assertEqual(len(x), 1) 965 self.assertEqual(dir(x[0]), dir(i)) 966 self.assertEqual(x[0].attr.keys(), [1]) 967 self.assertTrue(x[0].attr[1] is x) 968 969 def check_recursive_collection_and_inst(self, factory): 970 h = H() 971 y = factory([h]) 972 h.attr = y 973 for proto in protocols: 974 s = self.dumps(y, proto) 975 x = self.loads(s) 976 self.assertIsInstance(x, type(y)) 977 self.assertEqual(len(x), 1) 978 self.assertIsInstance(list(x)[0], H) 979 self.assertIs(list(x)[0].attr, x) 980 981 def test_recursive_list_and_inst(self): 982 self.check_recursive_collection_and_inst(list) 983 984 def test_recursive_tuple_and_inst(self): 985 self.check_recursive_collection_and_inst(tuple) 986 987 def test_recursive_dict_and_inst(self): 988 self.check_recursive_collection_and_inst(dict.fromkeys) 989 990 def test_recursive_set_and_inst(self): 991 self.check_recursive_collection_and_inst(set) 992 993 def test_recursive_frozenset_and_inst(self): 994 self.check_recursive_collection_and_inst(frozenset) 995 996 def test_recursive_list_subclass_and_inst(self): 997 self.check_recursive_collection_and_inst(MyList) 998 999 def test_recursive_tuple_subclass_and_inst(self): 1000 self.check_recursive_collection_and_inst(MyTuple) 1001 1002 def test_recursive_dict_subclass_and_inst(self): 1003 self.check_recursive_collection_and_inst(MyDict.fromkeys) 1004 1005 if have_unicode: 1006 def test_unicode(self): 1007 endcases = [u'', u'<\\u>', u'<\\\u1234>', u'<\n>', 1008 u'<\\>', u'<\\\U00012345>', 1009 # surrogates 1010 u'<\udc80>'] 1011 for proto in protocols: 1012 for u in endcases: 1013 p = self.dumps(u, proto) 1014 u2 = self.loads(p) 1015 self.assertEqual(u2, u) 1016 1017 def test_unicode_high_plane(self): 1018 t = u'\U00012345' 1019 for proto in protocols: 1020 p = self.dumps(t, proto) 1021 t2 = self.loads(p) 1022 self.assertEqual(t2, t) 1023 1024 def test_ints(self): 1025 import sys 1026 for proto in protocols: 1027 n = sys.maxint 1028 while n: 1029 for expected in (-n, n): 1030 s = self.dumps(expected, proto) 1031 n2 = self.loads(s) 1032 self.assertEqual(expected, n2) 1033 n = n >> 1 1034 1035 def test_long(self): 1036 for proto in protocols: 1037 # 256 bytes is where LONG4 begins. 1038 for nbits in 1, 8, 8*254, 8*255, 8*256, 8*257: 1039 nbase = 1L << nbits 1040 for npos in nbase-1, nbase, nbase+1: 1041 for n in npos, -npos: 1042 pickle = self.dumps(n, proto) 1043 got = self.loads(pickle) 1044 self.assertEqual(n, got) 1045 # Try a monster. This is quadratic-time in protos 0 & 1, so don't 1046 # bother with those. 1047 nbase = long("deadbeeffeedface", 16) 1048 nbase += nbase << 1000000 1049 for n in nbase, -nbase: 1050 p = self.dumps(n, 2) 1051 got = self.loads(p) 1052 self.assertEqual(n, got) 1053 1054 def test_float(self): 1055 test_values = [0.0, 4.94e-324, 1e-310, 7e-308, 6.626e-34, 0.1, 0.5, 1056 3.14, 263.44582062374053, 6.022e23, 1e30] 1057 test_values = test_values + [-x for x in test_values] 1058 for proto in protocols: 1059 for value in test_values: 1060 pickle = self.dumps(value, proto) 1061 got = self.loads(pickle) 1062 self.assertEqual(value, got) 1063 1064 @run_with_locale('LC_ALL', 'de_DE', 'fr_FR') 1065 def test_float_format(self): 1066 # make sure that floats are formatted locale independent 1067 self.assertEqual(self.dumps(1.2)[0:3], 'F1.') 1068 1069 def test_reduce(self): 1070 pass 1071 1072 def test_getinitargs(self): 1073 pass 1074 1075 def test_metaclass(self): 1076 a = use_metaclass() 1077 for proto in protocols: 1078 s = self.dumps(a, proto) 1079 b = self.loads(s) 1080 self.assertEqual(a.__class__, b.__class__) 1081 1082 def test_dynamic_class(self): 1083 a = create_dynamic_class("my_dynamic_class", (object,)) 1084 copy_reg.pickle(pickling_metaclass, pickling_metaclass.__reduce__) 1085 for proto in protocols: 1086 s = self.dumps(a, proto) 1087 b = self.loads(s) 1088 self.assertEqual(a, b) 1089 self.assertIs(a.__class__, b.__class__) 1090 1091 def test_structseq(self): 1092 import time 1093 import os 1094 1095 t = time.localtime() 1096 for proto in protocols: 1097 s = self.dumps(t, proto) 1098 u = self.loads(s) 1099 self.assertEqual(t, u) 1100 if hasattr(os, "stat"): 1101 t = os.stat(os.curdir) 1102 s = self.dumps(t, proto) 1103 u = self.loads(s) 1104 self.assertEqual(t, u) 1105 if hasattr(os, "statvfs"): 1106 t = os.statvfs(os.curdir) 1107 s = self.dumps(t, proto) 1108 u = self.loads(s) 1109 self.assertEqual(t, u) 1110 1111 # Tests for protocol 2 1112 1113 def test_proto(self): 1114 build_none = pickle.NONE + pickle.STOP 1115 for proto in protocols: 1116 expected = build_none 1117 if proto >= 2: 1118 expected = pickle.PROTO + chr(proto) + expected 1119 p = self.dumps(None, proto) 1120 self.assertEqual(p, expected) 1121 1122 oob = protocols[-1] + 1 # a future protocol 1123 badpickle = pickle.PROTO + chr(oob) + build_none 1124 try: 1125 self.loads(badpickle) 1126 except ValueError, detail: 1127 self.assertTrue(str(detail).startswith( 1128 "unsupported pickle protocol")) 1129 else: 1130 self.fail("expected bad protocol number to raise ValueError") 1131 1132 def test_long1(self): 1133 x = 12345678910111213141516178920L 1134 for proto in protocols: 1135 s = self.dumps(x, proto) 1136 y = self.loads(s) 1137 self.assertEqual(x, y) 1138 self.assertEqual(opcode_in_pickle(pickle.LONG1, s), proto >= 2) 1139 1140 def test_long4(self): 1141 x = 12345678910111213141516178920L << (256*8) 1142 for proto in protocols: 1143 s = self.dumps(x, proto) 1144 y = self.loads(s) 1145 self.assertEqual(x, y) 1146 self.assertEqual(opcode_in_pickle(pickle.LONG4, s), proto >= 2) 1147 1148 def test_short_tuples(self): 1149 # Map (proto, len(tuple)) to expected opcode. 1150 expected_opcode = {(0, 0): pickle.TUPLE, 1151 (0, 1): pickle.TUPLE, 1152 (0, 2): pickle.TUPLE, 1153 (0, 3): pickle.TUPLE, 1154 (0, 4): pickle.TUPLE, 1155 1156 (1, 0): pickle.EMPTY_TUPLE, 1157 (1, 1): pickle.TUPLE, 1158 (1, 2): pickle.TUPLE, 1159 (1, 3): pickle.TUPLE, 1160 (1, 4): pickle.TUPLE, 1161 1162 (2, 0): pickle.EMPTY_TUPLE, 1163 (2, 1): pickle.TUPLE1, 1164 (2, 2): pickle.TUPLE2, 1165 (2, 3): pickle.TUPLE3, 1166 (2, 4): pickle.TUPLE, 1167 } 1168 a = () 1169 b = (1,) 1170 c = (1, 2) 1171 d = (1, 2, 3) 1172 e = (1, 2, 3, 4) 1173 for proto in protocols: 1174 for x in a, b, c, d, e: 1175 s = self.dumps(x, proto) 1176 y = self.loads(s) 1177 self.assertEqual(x, y, (proto, x, s, y)) 1178 expected = expected_opcode[proto, len(x)] 1179 self.assertEqual(opcode_in_pickle(expected, s), True) 1180 1181 def test_singletons(self): 1182 # Map (proto, singleton) to expected opcode. 1183 expected_opcode = {(0, None): pickle.NONE, 1184 (1, None): pickle.NONE, 1185 (2, None): pickle.NONE, 1186 1187 (0, True): pickle.INT, 1188 (1, True): pickle.INT, 1189 (2, True): pickle.NEWTRUE, 1190 1191 (0, False): pickle.INT, 1192 (1, False): pickle.INT, 1193 (2, False): pickle.NEWFALSE, 1194 } 1195 for proto in protocols: 1196 for x in None, False, True: 1197 s = self.dumps(x, proto) 1198 y = self.loads(s) 1199 self.assertTrue(x is y, (proto, x, s, y)) 1200 expected = expected_opcode[proto, x] 1201 self.assertEqual(opcode_in_pickle(expected, s), True) 1202 1203 def test_newobj_tuple(self): 1204 x = MyTuple([1, 2, 3]) 1205 x.foo = 42 1206 x.bar = "hello" 1207 for proto in protocols: 1208 s = self.dumps(x, proto) 1209 y = self.loads(s) 1210 self.assertEqual(tuple(x), tuple(y)) 1211 self.assertEqual(x.__dict__, y.__dict__) 1212 1213 def test_newobj_list(self): 1214 x = MyList([1, 2, 3]) 1215 x.foo = 42 1216 x.bar = "hello" 1217 for proto in protocols: 1218 s = self.dumps(x, proto) 1219 y = self.loads(s) 1220 self.assertEqual(list(x), list(y)) 1221 self.assertEqual(x.__dict__, y.__dict__) 1222 1223 def test_newobj_generic(self): 1224 for proto in protocols: 1225 for C in myclasses: 1226 B = C.__base__ 1227 x = C(C.sample) 1228 x.foo = 42 1229 s = self.dumps(x, proto) 1230 y = self.loads(s) 1231 detail = (proto, C, B, x, y, type(y)) 1232 self.assertEqual(B(x), B(y), detail) 1233 self.assertEqual(x.__dict__, y.__dict__, detail) 1234 1235 def test_newobj_proxies(self): 1236 # NEWOBJ should use the __class__ rather than the raw type 1237 import weakref 1238 classes = myclasses[:] 1239 # Cannot create weakproxies to these classes 1240 for c in (MyInt, MyLong, MyStr, MyTuple): 1241 classes.remove(c) 1242 for proto in protocols: 1243 for C in classes: 1244 B = C.__base__ 1245 x = C(C.sample) 1246 x.foo = 42 1247 p = weakref.proxy(x) 1248 s = self.dumps(p, proto) 1249 y = self.loads(s) 1250 self.assertEqual(type(y), type(x)) # rather than type(p) 1251 detail = (proto, C, B, x, y, type(y)) 1252 self.assertEqual(B(x), B(y), detail) 1253 self.assertEqual(x.__dict__, y.__dict__, detail) 1254 1255 # Register a type with copy_reg, with extension code extcode. Pickle 1256 # an object of that type. Check that the resulting pickle uses opcode 1257 # (EXT[124]) under proto 2, and not in proto 1. 1258 1259 def produce_global_ext(self, extcode, opcode): 1260 e = ExtensionSaver(extcode) 1261 try: 1262 copy_reg.add_extension(__name__, "MyList", extcode) 1263 x = MyList([1, 2, 3]) 1264 x.foo = 42 1265 x.bar = "hello" 1266 1267 # Dump using protocol 1 for comparison. 1268 s1 = self.dumps(x, 1) 1269 self.assertIn(__name__, s1) 1270 self.assertIn("MyList", s1) 1271 self.assertEqual(opcode_in_pickle(opcode, s1), False) 1272 1273 y = self.loads(s1) 1274 self.assertEqual(list(x), list(y)) 1275 self.assertEqual(x.__dict__, y.__dict__) 1276 1277 # Dump using protocol 2 for test. 1278 s2 = self.dumps(x, 2) 1279 self.assertNotIn(__name__, s2) 1280 self.assertNotIn("MyList", s2) 1281 self.assertEqual(opcode_in_pickle(opcode, s2), True) 1282 1283 y = self.loads(s2) 1284 self.assertEqual(list(x), list(y)) 1285 self.assertEqual(x.__dict__, y.__dict__) 1286 1287 finally: 1288 e.restore() 1289 1290 def test_global_ext1(self): 1291 self.produce_global_ext(0x00000001, pickle.EXT1) # smallest EXT1 code 1292 self.produce_global_ext(0x000000ff, pickle.EXT1) # largest EXT1 code 1293 1294 def test_global_ext2(self): 1295 self.produce_global_ext(0x00000100, pickle.EXT2) # smallest EXT2 code 1296 self.produce_global_ext(0x0000ffff, pickle.EXT2) # largest EXT2 code 1297 self.produce_global_ext(0x0000abcd, pickle.EXT2) # check endianness 1298 1299 def test_global_ext4(self): 1300 self.produce_global_ext(0x00010000, pickle.EXT4) # smallest EXT4 code 1301 self.produce_global_ext(0x7fffffff, pickle.EXT4) # largest EXT4 code 1302 self.produce_global_ext(0x12abcdef, pickle.EXT4) # check endianness 1303 1304 def test_list_chunking(self): 1305 n = 10 # too small to chunk 1306 x = range(n) 1307 for proto in protocols: 1308 s = self.dumps(x, proto) 1309 y = self.loads(s) 1310 self.assertEqual(x, y) 1311 num_appends = count_opcode(pickle.APPENDS, s) 1312 self.assertEqual(num_appends, proto > 0) 1313 1314 n = 2500 # expect at least two chunks when proto > 0 1315 x = range(n) 1316 for proto in protocols: 1317 s = self.dumps(x, proto) 1318 y = self.loads(s) 1319 self.assertEqual(x, y) 1320 num_appends = count_opcode(pickle.APPENDS, s) 1321 if proto == 0: 1322 self.assertEqual(num_appends, 0) 1323 else: 1324 self.assertTrue(num_appends >= 2) 1325 1326 def test_dict_chunking(self): 1327 n = 10 # too small to chunk 1328 x = dict.fromkeys(range(n)) 1329 for proto in protocols: 1330 s = self.dumps(x, proto) 1331 y = self.loads(s) 1332 self.assertEqual(x, y) 1333 num_setitems = count_opcode(pickle.SETITEMS, s) 1334 self.assertEqual(num_setitems, proto > 0) 1335 1336 n = 2500 # expect at least two chunks when proto > 0 1337 x = dict.fromkeys(range(n)) 1338 for proto in protocols: 1339 s = self.dumps(x, proto) 1340 y = self.loads(s) 1341 self.assertEqual(x, y) 1342 num_setitems = count_opcode(pickle.SETITEMS, s) 1343 if proto == 0: 1344 self.assertEqual(num_setitems, 0) 1345 else: 1346 self.assertTrue(num_setitems >= 2) 1347 1348 def test_simple_newobj(self): 1349 x = SimpleNewObj.__new__(SimpleNewObj, 0xface) # avoid __init__ 1350 x.abc = 666 1351 for proto in protocols: 1352 s = self.dumps(x, proto) 1353 if proto < 1: 1354 self.assertIn('\nI64206', s) # INT 1355 else: 1356 self.assertIn('M\xce\xfa', s) # BININT2 1357 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), proto >= 2) 1358 y = self.loads(s) # will raise TypeError if __init__ called 1359 self.assertEqual(y.abc, 666) 1360 self.assertEqual(x.__dict__, y.__dict__) 1361 1362 def test_complex_newobj(self): 1363 x = ComplexNewObj.__new__(ComplexNewObj, 0xface) # avoid __init__ 1364 x.abc = 666 1365 for proto in protocols: 1366 s = self.dumps(x, proto) 1367 if proto < 1: 1368 self.assertIn('\nI64206', s) # INT 1369 elif proto < 2: 1370 self.assertIn('M\xce\xfa', s) # BININT2 1371 else: 1372 self.assertIn('U\x04FACE', s) # SHORT_BINSTRING 1373 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), proto >= 2) 1374 y = self.loads(s) # will raise TypeError if __init__ called 1375 self.assertEqual(y.abc, 666) 1376 self.assertEqual(x.__dict__, y.__dict__) 1377 1378 def test_newobj_list_slots(self): 1379 x = SlotList([1, 2, 3]) 1380 x.foo = 42 1381 x.bar = "hello" 1382 s = self.dumps(x, 2) 1383 y = self.loads(s) 1384 self.assertEqual(list(x), list(y)) 1385 self.assertEqual(x.__dict__, y.__dict__) 1386 self.assertEqual(x.foo, y.foo) 1387 self.assertEqual(x.bar, y.bar) 1388 1389 def test_reduce_overrides_default_reduce_ex(self): 1390 for proto in protocols: 1391 x = REX_one() 1392 self.assertEqual(x._reduce_called, 0) 1393 s = self.dumps(x, proto) 1394 self.assertEqual(x._reduce_called, 1) 1395 y = self.loads(s) 1396 self.assertEqual(y._reduce_called, 0) 1397 1398 def test_reduce_ex_called(self): 1399 for proto in protocols: 1400 x = REX_two() 1401 self.assertEqual(x._proto, None) 1402 s = self.dumps(x, proto) 1403 self.assertEqual(x._proto, proto) 1404 y = self.loads(s) 1405 self.assertEqual(y._proto, None) 1406 1407 def test_reduce_ex_overrides_reduce(self): 1408 for proto in protocols: 1409 x = REX_three() 1410 self.assertEqual(x._proto, None) 1411 s = self.dumps(x, proto) 1412 self.assertEqual(x._proto, proto) 1413 y = self.loads(s) 1414 self.assertEqual(y._proto, None) 1415 1416 def test_reduce_ex_calls_base(self): 1417 for proto in protocols: 1418 x = REX_four() 1419 self.assertEqual(x._proto, None) 1420 s = self.dumps(x, proto) 1421 self.assertEqual(x._proto, proto) 1422 y = self.loads(s) 1423 self.assertEqual(y._proto, proto) 1424 1425 def test_reduce_calls_base(self): 1426 for proto in protocols: 1427 x = REX_five() 1428 self.assertEqual(x._reduce_called, 0) 1429 s = self.dumps(x, proto) 1430 self.assertEqual(x._reduce_called, 1) 1431 y = self.loads(s) 1432 self.assertEqual(y._reduce_called, 1) 1433 1434 @no_tracing 1435 def test_bad_getattr(self): 1436 # Issue #3514: crash when there is an infinite loop in __getattr__ 1437 x = BadGetattr() 1438 for proto in protocols: 1439 self.assertRaises(RuntimeError, self.dumps, x, proto) 1440 1441 def test_reduce_bad_iterator(self): 1442 # Issue4176: crash when 4th and 5th items of __reduce__() 1443 # are not iterators 1444 class C(object): 1445 def __reduce__(self): 1446 # 4th item is not an iterator 1447 return list, (), None, [], None 1448 class D(object): 1449 def __reduce__(self): 1450 # 5th item is not an iterator 1451 return dict, (), None, None, [] 1452 1453 # Protocol 0 in Python implementation is less strict and also accepts 1454 # iterables. 1455 for proto in protocols: 1456 try: 1457 self.dumps(C(), proto) 1458 except (AttributeError, pickle.PicklingError, cPickle.PicklingError): 1459 pass 1460 try: 1461 self.dumps(D(), proto) 1462 except (AttributeError, pickle.PicklingError, cPickle.PicklingError): 1463 pass 1464 1465 def test_many_puts_and_gets(self): 1466 # Test that internal data structures correctly deal with lots of 1467 # puts/gets. 1468 keys = ("aaa" + str(i) for i in xrange(100)) 1469 large_dict = dict((k, [4, 5, 6]) for k in keys) 1470 obj = [dict(large_dict), dict(large_dict), dict(large_dict)] 1471 1472 for proto in protocols: 1473 dumped = self.dumps(obj, proto) 1474 loaded = self.loads(dumped) 1475 self.assertEqual(loaded, obj, 1476 "Failed protocol %d: %r != %r" 1477 % (proto, obj, loaded)) 1478 1479 def test_attribute_name_interning(self): 1480 # Test that attribute names of pickled objects are interned when 1481 # unpickling. 1482 for proto in protocols: 1483 x = C() 1484 x.foo = 42 1485 x.bar = "hello" 1486 s = self.dumps(x, proto) 1487 y = self.loads(s) 1488 x_keys = sorted(x.__dict__) 1489 y_keys = sorted(y.__dict__) 1490 for x_key, y_key in zip(x_keys, y_keys): 1491 self.assertIs(x_key, y_key) 1492 1493 def test_large_pickles(self): 1494 # Test the correctness of internal buffering routines when handling 1495 # large data. 1496 for proto in protocols: 1497 data = (1, min, 'xy' * (30 * 1024), len) 1498 dumped = self.dumps(data, proto) 1499 loaded = self.loads(dumped) 1500 self.assertEqual(len(loaded), len(data)) 1501 self.assertEqual(loaded, data) 1502 1503 def _check_pickling_with_opcode(self, obj, opcode, proto): 1504 pickled = self.dumps(obj, proto) 1505 self.assertTrue(opcode_in_pickle(opcode, pickled)) 1506 unpickled = self.loads(pickled) 1507 self.assertEqual(obj, unpickled) 1508 1509 def test_appends_on_non_lists(self): 1510 # Issue #17720 1511 obj = REX_six([1, 2, 3]) 1512 for proto in protocols: 1513 if proto == 0: 1514 self._check_pickling_with_opcode(obj, pickle.APPEND, proto) 1515 else: 1516 self._check_pickling_with_opcode(obj, pickle.APPENDS, proto) 1517 1518 def test_setitems_on_non_dicts(self): 1519 obj = REX_seven({1: -1, 2: -2, 3: -3}) 1520 for proto in protocols: 1521 if proto == 0: 1522 self._check_pickling_with_opcode(obj, pickle.SETITEM, proto) 1523 else: 1524 self._check_pickling_with_opcode(obj, pickle.SETITEMS, proto) 1525 1526 1527 # Test classes for reduce_ex 1528 1529 class REX_one(object): 1530 _reduce_called = 0 1531 def __reduce__(self): 1532 self._reduce_called = 1 1533 return REX_one, () 1534 # No __reduce_ex__ here, but inheriting it from object 1535 1536 class REX_two(object): 1537 _proto = None 1538 def __reduce_ex__(self, proto): 1539 self._proto = proto 1540 return REX_two, () 1541 # No __reduce__ here, but inheriting it from object 1542 1543 class REX_three(object): 1544 _proto = None 1545 def __reduce_ex__(self, proto): 1546 self._proto = proto 1547 return REX_two, () 1548 def __reduce__(self): 1549 raise TestFailed, "This __reduce__ shouldn't be called" 1550 1551 class REX_four(object): 1552 _proto = None 1553 def __reduce_ex__(self, proto): 1554 self._proto = proto 1555 return object.__reduce_ex__(self, proto) 1556 # Calling base class method should succeed 1557 1558 class REX_five(object): 1559 _reduce_called = 0 1560 def __reduce__(self): 1561 self._reduce_called = 1 1562 return object.__reduce__(self) 1563 # This one used to fail with infinite recursion 1564 1565 class REX_six(object): 1566 """This class is used to check the 4th argument (list iterator) of 1567 the reduce protocol. 1568 """ 1569 def __init__(self, items=None): 1570 if items is None: 1571 items = [] 1572 self.items = items 1573 def __eq__(self, other): 1574 return type(self) is type(other) and self.items == other.items 1575 __hash__ = None 1576 def append(self, item): 1577 self.items.append(item) 1578 def extend(self, items): 1579 for item in items: 1580 self.append(item) 1581 def __reduce__(self): 1582 return type(self), (), None, iter(self.items), None 1583 1584 class REX_seven(object): 1585 """This class is used to check the 5th argument (dict iterator) of 1586 the reduce protocol. 1587 """ 1588 def __init__(self, table=None): 1589 if table is None: 1590 table = {} 1591 self.table = table 1592 def __eq__(self, other): 1593 return type(self) is type(other) and self.table == other.table 1594 __hash__ = None 1595 def __setitem__(self, key, value): 1596 self.table[key] = value 1597 def __reduce__(self): 1598 return type(self), (), None, None, iter(self.table.items()) 1599 1600 # Test classes for newobj 1601 1602 class MyInt(int): 1603 sample = 1 1604 1605 class MyLong(long): 1606 sample = 1L 1607 1608 class MyFloat(float): 1609 sample = 1.0 1610 1611 class MyComplex(complex): 1612 sample = 1.0 + 0.0j 1613 1614 class MyStr(str): 1615 sample = "hello" 1616 1617 class MyUnicode(unicode): 1618 sample = u"hello \u1234" 1619 1620 class MyTuple(tuple): 1621 sample = (1, 2, 3) 1622 1623 class MyList(list): 1624 sample = [1, 2, 3] 1625 1626 class MyDict(dict): 1627 sample = {"a": 1, "b": 2} 1628 1629 myclasses = [MyInt, MyLong, MyFloat, 1630 MyComplex, 1631 MyStr, MyUnicode, 1632 MyTuple, MyList, MyDict] 1633 1634 1635 class SlotList(MyList): 1636 __slots__ = ["foo"] 1637 1638 class SimpleNewObj(int): 1639 def __init__(self, *args, **kwargs): 1640 # raise an error, to make sure this isn't called 1641 raise TypeError("SimpleNewObj.__init__() didn't expect to get called") 1642 def __eq__(self, other): 1643 return int(self) == int(other) and self.__dict__ == other.__dict__ 1644 __hash__ = None 1645 1646 class ComplexNewObj(SimpleNewObj): 1647 def __getnewargs__(self): 1648 return ('%X' % self, 16) 1649 1650 class BadGetattr: 1651 def __getattr__(self, key): 1652 self.foo 1653 1654 class AbstractPickleModuleTests(unittest.TestCase): 1655 1656 def test_dump_closed_file(self): 1657 import os 1658 f = open(TESTFN, "w") 1659 try: 1660 f.close() 1661 self.assertRaises(ValueError, self.module.dump, 123, f) 1662 finally: 1663 os.remove(TESTFN) 1664 1665 def test_load_closed_file(self): 1666 import os 1667 f = open(TESTFN, "w") 1668 try: 1669 f.close() 1670 self.assertRaises(ValueError, self.module.dump, 123, f) 1671 finally: 1672 os.remove(TESTFN) 1673 1674 def test_load_from_and_dump_to_file(self): 1675 stream = cStringIO.StringIO() 1676 data = [123, {}, 124] 1677 self.module.dump(data, stream) 1678 stream.seek(0) 1679 unpickled = self.module.load(stream) 1680 self.assertEqual(unpickled, data) 1681 1682 def test_highest_protocol(self): 1683 # Of course this needs to be changed when HIGHEST_PROTOCOL changes. 1684 self.assertEqual(self.module.HIGHEST_PROTOCOL, 2) 1685 1686 def test_callapi(self): 1687 f = cStringIO.StringIO() 1688 # With and without keyword arguments 1689 self.module.dump(123, f, -1) 1690 self.module.dump(123, file=f, protocol=-1) 1691 self.module.dumps(123, -1) 1692 self.module.dumps(123, protocol=-1) 1693 self.module.Pickler(f, -1) 1694 self.module.Pickler(f, protocol=-1) 1695 1696 def test_incomplete_input(self): 1697 s = StringIO.StringIO("X''.") 1698 self.assertRaises(EOFError, self.module.load, s) 1699 1700 def test_restricted(self): 1701 # issue7128: cPickle failed in restricted mode 1702 builtins = {self.module.__name__: self.module, 1703 '__import__': __import__} 1704 d = {} 1705 teststr = "def f(): {0}.dumps(0)".format(self.module.__name__) 1706 exec teststr in {'__builtins__': builtins}, d 1707 d['f']() 1708 1709 def test_bad_input(self): 1710 # Test issue4298 1711 s = '\x58\0\0\0\x54' 1712 self.assertRaises(EOFError, self.module.loads, s) 1713 1714 1715 class AbstractPersistentPicklerTests(unittest.TestCase): 1716 1717 # This class defines persistent_id() and persistent_load() 1718 # functions that should be used by the pickler. All even integers 1719 # are pickled using persistent ids. 1720 1721 def persistent_id(self, object): 1722 if isinstance(object, int) and object % 2 == 0: 1723 self.id_count += 1 1724 return str(object) 1725 elif object == "test_false_value": 1726 self.false_count += 1 1727 return "" 1728 else: 1729 return None 1730 1731 def persistent_load(self, oid): 1732 if not oid: 1733 self.load_false_count += 1 1734 return "test_false_value" 1735 else: 1736 self.load_count += 1 1737 object = int(oid) 1738 assert object % 2 == 0 1739 return object 1740 1741 def test_persistence(self): 1742 L = range(10) + ["test_false_value"] 1743 for proto in protocols: 1744 self.id_count = 0 1745 self.false_count = 0 1746 self.load_false_count = 0 1747 self.load_count = 0 1748 self.assertEqual(self.loads(self.dumps(L, proto)), L) 1749 self.assertEqual(self.id_count, 5) 1750 self.assertEqual(self.false_count, 1) 1751 self.assertEqual(self.load_count, 5) 1752 self.assertEqual(self.load_false_count, 1) 1753 1754 class AbstractPicklerUnpicklerObjectTests(unittest.TestCase): 1755 1756 pickler_class = None 1757 unpickler_class = None 1758 1759 def setUp(self): 1760 assert self.pickler_class 1761 assert self.unpickler_class 1762 1763 def test_clear_pickler_memo(self): 1764 # To test whether clear_memo() has any effect, we pickle an object, 1765 # then pickle it again without clearing the memo; the two serialized 1766 # forms should be different. If we clear_memo() and then pickle the 1767 # object again, the third serialized form should be identical to the 1768 # first one we obtained. 1769 data = ["abcdefg", "abcdefg", 44] 1770 f = cStringIO.StringIO() 1771 pickler = self.pickler_class(f) 1772 1773 pickler.dump(data) 1774 first_pickled = f.getvalue() 1775 1776 # Reset StringIO object. 1777 f.seek(0) 1778 f.truncate() 1779 1780 pickler.dump(data) 1781 second_pickled = f.getvalue() 1782 1783 # Reset the Pickler and StringIO objects. 1784 pickler.clear_memo() 1785 f.seek(0) 1786 f.truncate() 1787 1788 pickler.dump(data) 1789 third_pickled = f.getvalue() 1790 1791 self.assertNotEqual(first_pickled, second_pickled) 1792 self.assertEqual(first_pickled, third_pickled) 1793 1794 def test_priming_pickler_memo(self): 1795 # Verify that we can set the Pickler's memo attribute. 1796 data = ["abcdefg", "abcdefg", 44] 1797 f = cStringIO.StringIO() 1798 pickler = self.pickler_class(f) 1799 1800 pickler.dump(data) 1801 first_pickled = f.getvalue() 1802 1803 f = cStringIO.StringIO() 1804 primed = self.pickler_class(f) 1805 primed.memo = pickler.memo 1806 1807 primed.dump(data) 1808 primed_pickled = f.getvalue() 1809 1810 self.assertNotEqual(first_pickled, primed_pickled) 1811 1812 def test_priming_unpickler_memo(self): 1813 # Verify that we can set the Unpickler's memo attribute. 1814 data = ["abcdefg", "abcdefg", 44] 1815 f = cStringIO.StringIO() 1816 pickler = self.pickler_class(f) 1817 1818 pickler.dump(data) 1819 first_pickled = f.getvalue() 1820 1821 f = cStringIO.StringIO() 1822 primed = self.pickler_class(f) 1823 primed.memo = pickler.memo 1824 1825 primed.dump(data) 1826 primed_pickled = f.getvalue() 1827 1828 unpickler = self.unpickler_class(cStringIO.StringIO(first_pickled)) 1829 unpickled_data1 = unpickler.load() 1830 1831 self.assertEqual(unpickled_data1, data) 1832 1833 primed = self.unpickler_class(cStringIO.StringIO(primed_pickled)) 1834 primed.memo = unpickler.memo 1835 unpickled_data2 = primed.load() 1836 1837 primed.memo.clear() 1838 1839 self.assertEqual(unpickled_data2, data) 1840 self.assertTrue(unpickled_data2 is unpickled_data1) 1841 1842 def test_reusing_unpickler_objects(self): 1843 data1 = ["abcdefg", "abcdefg", 44] 1844 f = cStringIO.StringIO() 1845 pickler = self.pickler_class(f) 1846 pickler.dump(data1) 1847 pickled1 = f.getvalue() 1848 1849 data2 = ["abcdefg", 44, 44] 1850 f = cStringIO.StringIO() 1851 pickler = self.pickler_class(f) 1852 pickler.dump(data2) 1853 pickled2 = f.getvalue() 1854 1855 f = cStringIO.StringIO() 1856 f.write(pickled1) 1857 f.seek(0) 1858 unpickler = self.unpickler_class(f) 1859 self.assertEqual(unpickler.load(), data1) 1860 1861 f.seek(0) 1862 f.truncate() 1863 f.write(pickled2) 1864 f.seek(0) 1865 self.assertEqual(unpickler.load(), data2) 1866 1867 def _check_multiple_unpicklings(self, ioclass, seekable): 1868 for proto in protocols: 1869 data1 = [(x, str(x)) for x in xrange(2000)] + ["abcde", len] 1870 f = ioclass() 1871 pickler = self.pickler_class(f, protocol=proto) 1872 pickler.dump(data1) 1873 pickled = f.getvalue() 1874 1875 N = 5 1876 f = ioclass(pickled * N) 1877 unpickler = self.unpickler_class(f) 1878 for i in xrange(N): 1879 if seekable: 1880 pos = f.tell() 1881 self.assertEqual(unpickler.load(), data1) 1882 if seekable: 1883 self.assertEqual(f.tell(), pos + len(pickled)) 1884 self.assertRaises(EOFError, unpickler.load) 1885 1886 def test_multiple_unpicklings_seekable(self): 1887 self._check_multiple_unpicklings(StringIO.StringIO, True) 1888 1889 def test_multiple_unpicklings_unseekable(self): 1890 self._check_multiple_unpicklings(UnseekableIO, False) 1891 1892 def test_unpickling_buffering_readline(self): 1893 # Issue #12687: the unpickler's buffering logic could fail with 1894 # text mode opcodes. 1895 import io 1896 data = list(xrange(10)) 1897 for proto in protocols: 1898 for buf_size in xrange(1, 11): 1899 f = io.BufferedRandom(io.BytesIO(), buffer_size=buf_size) 1900 pickler = self.pickler_class(f, protocol=proto) 1901 pickler.dump(data) 1902 f.seek(0) 1903 unpickler = self.unpickler_class(f) 1904 self.assertEqual(unpickler.load(), data) 1905 1906 1907 class BigmemPickleTests(unittest.TestCase): 1908 1909 # Memory requirements: 1 byte per character for input strings, 1 byte 1910 # for pickled data, 1 byte for unpickled strings, 1 byte for internal 1911 # buffer and 1 byte of free space for resizing of internal buffer. 1912 1913 @precisionbigmemtest(size=_2G + 100*_1M, memuse=5) 1914 def test_huge_strlist(self, size): 1915 chunksize = 2**20 1916 data = [] 1917 while size > chunksize: 1918 data.append('x' * chunksize) 1919 size -= chunksize 1920 chunksize += 1 1921 data.append('y' * size) 1922 1923 try: 1924 for proto in protocols: 1925 try: 1926 pickled = self.dumps(data, proto) 1927 res = self.loads(pickled) 1928 self.assertEqual(res, data) 1929 finally: 1930 res = None 1931 pickled = None 1932 finally: 1933 data = None 1934
