1 /* zlibmodule.c -- gzip-compatible data compression */ 2 /* See http://zlib.net/ */ 3 4 /* Windows users: read Python's PCbuild\readme.txt */ 5 6 #define PY_SSIZE_T_CLEAN 7 8 #include "Python.h" 9 #include "structmember.h" 10 #include "zlib.h" 11 12 13 #ifdef WITH_THREAD 14 #include "pythread.h" 15 #define ENTER_ZLIB(obj) \ 16 Py_BEGIN_ALLOW_THREADS; \ 17 PyThread_acquire_lock((obj)->lock, 1); \ 18 Py_END_ALLOW_THREADS; 19 #define LEAVE_ZLIB(obj) PyThread_release_lock((obj)->lock); 20 #else 21 #define ENTER_ZLIB(obj) 22 #define LEAVE_ZLIB(obj) 23 #endif 24 25 #if defined(ZLIB_VERNUM) && ZLIB_VERNUM >= 0x1221 26 # define AT_LEAST_ZLIB_1_2_2_1 27 #endif 28 29 /* The following parameters are copied from zutil.h, version 0.95 */ 30 #define DEFLATED 8 31 #if MAX_MEM_LEVEL >= 8 32 # define DEF_MEM_LEVEL 8 33 #else 34 # define DEF_MEM_LEVEL MAX_MEM_LEVEL 35 #endif 36 37 /* Initial buffer size. */ 38 #define DEF_BUF_SIZE (16*1024) 39 40 static PyTypeObject Comptype; 41 static PyTypeObject Decomptype; 42 43 static PyObject *ZlibError; 44 45 typedef struct 46 { 47 PyObject_HEAD 48 z_stream zst; 49 PyObject *unused_data; 50 PyObject *unconsumed_tail; 51 char eof; 52 int is_initialised; 53 PyObject *zdict; 54 #ifdef WITH_THREAD 55 PyThread_type_lock lock; 56 #endif 57 } compobject; 58 59 static void 60 zlib_error(z_stream zst, int err, const char *msg) 61 { 62 const char *zmsg = Z_NULL; 63 /* In case of a version mismatch, zst.msg won't be initialized. 64 Check for this case first, before looking at zst.msg. */ 65 if (err == Z_VERSION_ERROR) 66 zmsg = "library version mismatch"; 67 if (zmsg == Z_NULL) 68 zmsg = zst.msg; 69 if (zmsg == Z_NULL) { 70 switch (err) { 71 case Z_BUF_ERROR: 72 zmsg = "incomplete or truncated stream"; 73 break; 74 case Z_STREAM_ERROR: 75 zmsg = "inconsistent stream state"; 76 break; 77 case Z_DATA_ERROR: 78 zmsg = "invalid input data"; 79 break; 80 } 81 } 82 if (zmsg == Z_NULL) 83 PyErr_Format(ZlibError, "Error %d %s", err, msg); 84 else 85 PyErr_Format(ZlibError, "Error %d %s: %.200s", err, msg, zmsg); 86 } 87 88 /*[clinic input] 89 module zlib 90 class zlib.Compress "compobject *" "&Comptype" 91 class zlib.Decompress "compobject *" "&Decomptype" 92 [clinic start generated code]*/ 93 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=093935115c3e3158]*/ 94 95 static compobject * 96 newcompobject(PyTypeObject *type) 97 { 98 compobject *self; 99 self = PyObject_New(compobject, type); 100 if (self == NULL) 101 return NULL; 102 self->eof = 0; 103 self->is_initialised = 0; 104 self->zdict = NULL; 105 self->unused_data = PyBytes_FromStringAndSize("", 0); 106 if (self->unused_data == NULL) { 107 Py_DECREF(self); 108 return NULL; 109 } 110 self->unconsumed_tail = PyBytes_FromStringAndSize("", 0); 111 if (self->unconsumed_tail == NULL) { 112 Py_DECREF(self); 113 return NULL; 114 } 115 #ifdef WITH_THREAD 116 self->lock = PyThread_allocate_lock(); 117 if (self->lock == NULL) { 118 Py_DECREF(self); 119 PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); 120 return NULL; 121 } 122 #endif 123 return self; 124 } 125 126 static void* 127 PyZlib_Malloc(voidpf ctx, uInt items, uInt size) 128 { 129 if (items > (size_t)PY_SSIZE_T_MAX / size) 130 return NULL; 131 /* PyMem_Malloc() cannot be used: the GIL is not held when 132 inflate() and deflate() are called */ 133 return PyMem_RawMalloc(items * size); 134 } 135 136 static void 137 PyZlib_Free(voidpf ctx, void *ptr) 138 { 139 PyMem_RawFree(ptr); 140 } 141 142 static void 143 arrange_input_buffer(z_stream *zst, Py_ssize_t *remains) 144 { 145 zst->avail_in = Py_MIN((size_t)*remains, UINT_MAX); 146 *remains -= zst->avail_in; 147 } 148 149 static Py_ssize_t 150 arrange_output_buffer_with_maximum(z_stream *zst, PyObject **buffer, 151 Py_ssize_t length, 152 Py_ssize_t max_length) 153 { 154 Py_ssize_t occupied; 155 156 if (*buffer == NULL) { 157 if (!(*buffer = PyBytes_FromStringAndSize(NULL, length))) 158 return -1; 159 occupied = 0; 160 } 161 else { 162 occupied = zst->next_out - (Byte *)PyBytes_AS_STRING(*buffer); 163 164 if (length == occupied) { 165 Py_ssize_t new_length; 166 assert(length <= max_length); 167 /* can not scale the buffer over max_length */ 168 if (length == max_length) 169 return -2; 170 if (length <= (max_length >> 1)) 171 new_length = length << 1; 172 else 173 new_length = max_length; 174 if (_PyBytes_Resize(buffer, new_length) < 0) 175 return -1; 176 length = new_length; 177 } 178 } 179 180 zst->avail_out = Py_MIN((size_t)(length - occupied), UINT_MAX); 181 zst->next_out = (Byte *)PyBytes_AS_STRING(*buffer) + occupied; 182 183 return length; 184 } 185 186 static Py_ssize_t 187 arrange_output_buffer(z_stream *zst, PyObject **buffer, Py_ssize_t length) 188 { 189 Py_ssize_t ret; 190 191 ret = arrange_output_buffer_with_maximum(zst, buffer, length, 192 PY_SSIZE_T_MAX); 193 if (ret == -2) 194 PyErr_NoMemory(); 195 196 return ret; 197 } 198 199 /*[clinic input] 200 zlib.compress 201 202 data: Py_buffer 203 Binary data to be compressed. 204 / 205 level: int(c_default="Z_DEFAULT_COMPRESSION") = Z_DEFAULT_COMPRESSION 206 Compression level, in 0-9 or -1. 207 208 Returns a bytes object containing compressed data. 209 [clinic start generated code]*/ 210 211 static PyObject * 212 zlib_compress_impl(PyObject *module, Py_buffer *data, int level) 213 /*[clinic end generated code: output=d80906d73f6294c8 input=638d54b6315dbed3]*/ 214 { 215 PyObject *RetVal = NULL; 216 Byte *ibuf; 217 Py_ssize_t ibuflen, obuflen = DEF_BUF_SIZE; 218 int err, flush; 219 z_stream zst; 220 221 ibuf = data->buf; 222 ibuflen = data->len; 223 224 zst.opaque = NULL; 225 zst.zalloc = PyZlib_Malloc; 226 zst.zfree = PyZlib_Free; 227 zst.next_in = ibuf; 228 err = deflateInit(&zst, level); 229 230 switch (err) { 231 case Z_OK: 232 break; 233 case Z_MEM_ERROR: 234 PyErr_SetString(PyExc_MemoryError, 235 "Out of memory while compressing data"); 236 goto error; 237 case Z_STREAM_ERROR: 238 PyErr_SetString(ZlibError, "Bad compression level"); 239 goto error; 240 default: 241 deflateEnd(&zst); 242 zlib_error(zst, err, "while compressing data"); 243 goto error; 244 } 245 246 do { 247 arrange_input_buffer(&zst, &ibuflen); 248 flush = ibuflen == 0 ? Z_FINISH : Z_NO_FLUSH; 249 250 do { 251 obuflen = arrange_output_buffer(&zst, &RetVal, obuflen); 252 if (obuflen < 0) { 253 deflateEnd(&zst); 254 goto error; 255 } 256 257 Py_BEGIN_ALLOW_THREADS 258 err = deflate(&zst, flush); 259 Py_END_ALLOW_THREADS 260 261 if (err == Z_STREAM_ERROR) { 262 deflateEnd(&zst); 263 zlib_error(zst, err, "while compressing data"); 264 goto error; 265 } 266 267 } while (zst.avail_out == 0); 268 assert(zst.avail_in == 0); 269 270 } while (flush != Z_FINISH); 271 assert(err == Z_STREAM_END); 272 273 err = deflateEnd(&zst); 274 if (err == Z_OK) { 275 if (_PyBytes_Resize(&RetVal, zst.next_out - 276 (Byte *)PyBytes_AS_STRING(RetVal)) < 0) 277 goto error; 278 return RetVal; 279 } 280 else 281 zlib_error(zst, err, "while finishing compression"); 282 error: 283 Py_XDECREF(RetVal); 284 return NULL; 285 } 286 287 /*[python input] 288 289 class ssize_t_converter(CConverter): 290 type = 'Py_ssize_t' 291 converter = 'ssize_t_converter' 292 c_ignored_default = "0" 293 294 [python start generated code]*/ 295 /*[python end generated code: output=da39a3ee5e6b4b0d input=5f34ba1b394cb8e7]*/ 296 297 static int 298 ssize_t_converter(PyObject *obj, void *ptr) 299 { 300 PyObject *long_obj; 301 Py_ssize_t val; 302 303 long_obj = (PyObject *)_PyLong_FromNbInt(obj); 304 if (long_obj == NULL) { 305 return 0; 306 } 307 val = PyLong_AsSsize_t(long_obj); 308 Py_DECREF(long_obj); 309 if (val == -1 && PyErr_Occurred()) { 310 return 0; 311 } 312 *(Py_ssize_t *)ptr = val; 313 return 1; 314 } 315 316 /*[clinic input] 317 zlib.decompress 318 319 data: Py_buffer 320 Compressed data. 321 / 322 wbits: int(c_default="MAX_WBITS") = MAX_WBITS 323 The window buffer size and container format. 324 bufsize: ssize_t(c_default="DEF_BUF_SIZE") = DEF_BUF_SIZE 325 The initial output buffer size. 326 327 Returns a bytes object containing the uncompressed data. 328 [clinic start generated code]*/ 329 330 static PyObject * 331 zlib_decompress_impl(PyObject *module, Py_buffer *data, int wbits, 332 Py_ssize_t bufsize) 333 /*[clinic end generated code: output=77c7e35111dc8c42 input=21960936208e9a5b]*/ 334 { 335 PyObject *RetVal = NULL; 336 Byte *ibuf; 337 Py_ssize_t ibuflen; 338 int err, flush; 339 z_stream zst; 340 341 if (bufsize < 0) { 342 PyErr_SetString(PyExc_ValueError, "bufsize must be non-negative"); 343 return NULL; 344 } else if (bufsize == 0) { 345 bufsize = 1; 346 } 347 348 ibuf = data->buf; 349 ibuflen = data->len; 350 351 zst.opaque = NULL; 352 zst.zalloc = PyZlib_Malloc; 353 zst.zfree = PyZlib_Free; 354 zst.avail_in = 0; 355 zst.next_in = ibuf; 356 err = inflateInit2(&zst, wbits); 357 358 switch (err) { 359 case Z_OK: 360 break; 361 case Z_MEM_ERROR: 362 PyErr_SetString(PyExc_MemoryError, 363 "Out of memory while decompressing data"); 364 goto error; 365 default: 366 inflateEnd(&zst); 367 zlib_error(zst, err, "while preparing to decompress data"); 368 goto error; 369 } 370 371 do { 372 arrange_input_buffer(&zst, &ibuflen); 373 flush = ibuflen == 0 ? Z_FINISH : Z_NO_FLUSH; 374 375 do { 376 bufsize = arrange_output_buffer(&zst, &RetVal, bufsize); 377 if (bufsize < 0) { 378 inflateEnd(&zst); 379 goto error; 380 } 381 382 Py_BEGIN_ALLOW_THREADS 383 err = inflate(&zst, flush); 384 Py_END_ALLOW_THREADS 385 386 switch (err) { 387 case Z_OK: /* fall through */ 388 case Z_BUF_ERROR: /* fall through */ 389 case Z_STREAM_END: 390 break; 391 case Z_MEM_ERROR: 392 inflateEnd(&zst); 393 PyErr_SetString(PyExc_MemoryError, 394 "Out of memory while decompressing data"); 395 goto error; 396 default: 397 inflateEnd(&zst); 398 zlib_error(zst, err, "while decompressing data"); 399 goto error; 400 } 401 402 } while (zst.avail_out == 0); 403 404 } while (err != Z_STREAM_END && ibuflen != 0); 405 406 407 if (err != Z_STREAM_END) { 408 inflateEnd(&zst); 409 zlib_error(zst, err, "while decompressing data"); 410 goto error; 411 } 412 413 err = inflateEnd(&zst); 414 if (err != Z_OK) { 415 zlib_error(zst, err, "while finishing decompression"); 416 goto error; 417 } 418 419 if (_PyBytes_Resize(&RetVal, zst.next_out - 420 (Byte *)PyBytes_AS_STRING(RetVal)) < 0) 421 goto error; 422 423 return RetVal; 424 425 error: 426 Py_XDECREF(RetVal); 427 return NULL; 428 } 429 430 /*[clinic input] 431 zlib.compressobj 432 433 level: int(c_default="Z_DEFAULT_COMPRESSION") = Z_DEFAULT_COMPRESSION 434 The compression level (an integer in the range 0-9 or -1; default is 435 currently equivalent to 6). Higher compression levels are slower, 436 but produce smaller results. 437 method: int(c_default="DEFLATED") = DEFLATED 438 The compression algorithm. If given, this must be DEFLATED. 439 wbits: int(c_default="MAX_WBITS") = MAX_WBITS 440 +9 to +15: The base-two logarithm of the window size. Include a zlib 441 container. 442 -9 to -15: Generate a raw stream. 443 +25 to +31: Include a gzip container. 444 memLevel: int(c_default="DEF_MEM_LEVEL") = DEF_MEM_LEVEL 445 Controls the amount of memory used for internal compression state. 446 Valid values range from 1 to 9. Higher values result in higher memory 447 usage, faster compression, and smaller output. 448 strategy: int(c_default="Z_DEFAULT_STRATEGY") = Z_DEFAULT_STRATEGY 449 Used to tune the compression algorithm. Possible values are 450 Z_DEFAULT_STRATEGY, Z_FILTERED, and Z_HUFFMAN_ONLY. 451 zdict: Py_buffer = None 452 The predefined compression dictionary - a sequence of bytes 453 containing subsequences that are likely to occur in the input data. 454 455 Return a compressor object. 456 [clinic start generated code]*/ 457 458 static PyObject * 459 zlib_compressobj_impl(PyObject *module, int level, int method, int wbits, 460 int memLevel, int strategy, Py_buffer *zdict) 461 /*[clinic end generated code: output=8b5bed9c8fc3814d input=2fa3d026f90ab8d5]*/ 462 { 463 compobject *self = NULL; 464 int err; 465 466 if (zdict->buf != NULL && (size_t)zdict->len > UINT_MAX) { 467 PyErr_SetString(PyExc_OverflowError, 468 "zdict length does not fit in an unsigned int"); 469 goto error; 470 } 471 472 self = newcompobject(&Comptype); 473 if (self == NULL) 474 goto error; 475 self->zst.opaque = NULL; 476 self->zst.zalloc = PyZlib_Malloc; 477 self->zst.zfree = PyZlib_Free; 478 self->zst.next_in = NULL; 479 self->zst.avail_in = 0; 480 err = deflateInit2(&self->zst, level, method, wbits, memLevel, strategy); 481 switch (err) { 482 case Z_OK: 483 self->is_initialised = 1; 484 if (zdict->buf == NULL) { 485 goto success; 486 } else { 487 err = deflateSetDictionary(&self->zst, 488 zdict->buf, (unsigned int)zdict->len); 489 switch (err) { 490 case Z_OK: 491 goto success; 492 case Z_STREAM_ERROR: 493 PyErr_SetString(PyExc_ValueError, "Invalid dictionary"); 494 goto error; 495 default: 496 PyErr_SetString(PyExc_ValueError, "deflateSetDictionary()"); 497 goto error; 498 } 499 } 500 case Z_MEM_ERROR: 501 PyErr_SetString(PyExc_MemoryError, 502 "Can't allocate memory for compression object"); 503 goto error; 504 case Z_STREAM_ERROR: 505 PyErr_SetString(PyExc_ValueError, "Invalid initialization option"); 506 goto error; 507 default: 508 zlib_error(self->zst, err, "while creating compression object"); 509 goto error; 510 } 511 512 error: 513 Py_CLEAR(self); 514 success: 515 return (PyObject *)self; 516 } 517 518 static int 519 set_inflate_zdict(compobject *self) 520 { 521 Py_buffer zdict_buf; 522 int err; 523 524 if (PyObject_GetBuffer(self->zdict, &zdict_buf, PyBUF_SIMPLE) == -1) { 525 return -1; 526 } 527 if ((size_t)zdict_buf.len > UINT_MAX) { 528 PyErr_SetString(PyExc_OverflowError, 529 "zdict length does not fit in an unsigned int"); 530 PyBuffer_Release(&zdict_buf); 531 return -1; 532 } 533 err = inflateSetDictionary(&self->zst, 534 zdict_buf.buf, (unsigned int)zdict_buf.len); 535 PyBuffer_Release(&zdict_buf); 536 if (err != Z_OK) { 537 zlib_error(self->zst, err, "while setting zdict"); 538 return -1; 539 } 540 return 0; 541 } 542 543 /*[clinic input] 544 zlib.decompressobj 545 546 wbits: int(c_default="MAX_WBITS") = MAX_WBITS 547 The window buffer size and container format. 548 zdict: object(c_default="NULL") = b'' 549 The predefined compression dictionary. This must be the same 550 dictionary as used by the compressor that produced the input data. 551 552 Return a decompressor object. 553 [clinic start generated code]*/ 554 555 static PyObject * 556 zlib_decompressobj_impl(PyObject *module, int wbits, PyObject *zdict) 557 /*[clinic end generated code: output=3069b99994f36906 input=d3832b8511fc977b]*/ 558 { 559 int err; 560 compobject *self; 561 562 if (zdict != NULL && !PyObject_CheckBuffer(zdict)) { 563 PyErr_SetString(PyExc_TypeError, 564 "zdict argument must support the buffer protocol"); 565 return NULL; 566 } 567 568 self = newcompobject(&Decomptype); 569 if (self == NULL) 570 return NULL; 571 self->zst.opaque = NULL; 572 self->zst.zalloc = PyZlib_Malloc; 573 self->zst.zfree = PyZlib_Free; 574 self->zst.next_in = NULL; 575 self->zst.avail_in = 0; 576 if (zdict != NULL) { 577 Py_INCREF(zdict); 578 self->zdict = zdict; 579 } 580 err = inflateInit2(&self->zst, wbits); 581 switch (err) { 582 case Z_OK: 583 self->is_initialised = 1; 584 if (self->zdict != NULL && wbits < 0) { 585 #ifdef AT_LEAST_ZLIB_1_2_2_1 586 if (set_inflate_zdict(self) < 0) { 587 Py_DECREF(self); 588 return NULL; 589 } 590 #else 591 PyErr_Format(ZlibError, 592 "zlib version %s does not allow raw inflate with dictionary", 593 ZLIB_VERSION); 594 Py_DECREF(self); 595 return NULL; 596 #endif 597 } 598 return (PyObject *)self; 599 case Z_STREAM_ERROR: 600 Py_DECREF(self); 601 PyErr_SetString(PyExc_ValueError, "Invalid initialization option"); 602 return NULL; 603 case Z_MEM_ERROR: 604 Py_DECREF(self); 605 PyErr_SetString(PyExc_MemoryError, 606 "Can't allocate memory for decompression object"); 607 return NULL; 608 default: 609 zlib_error(self->zst, err, "while creating decompression object"); 610 Py_DECREF(self); 611 return NULL; 612 } 613 } 614 615 static void 616 Dealloc(compobject *self) 617 { 618 #ifdef WITH_THREAD 619 PyThread_free_lock(self->lock); 620 #endif 621 Py_XDECREF(self->unused_data); 622 Py_XDECREF(self->unconsumed_tail); 623 Py_XDECREF(self->zdict); 624 PyObject_Del(self); 625 } 626 627 static void 628 Comp_dealloc(compobject *self) 629 { 630 if (self->is_initialised) 631 deflateEnd(&self->zst); 632 Dealloc(self); 633 } 634 635 static void 636 Decomp_dealloc(compobject *self) 637 { 638 if (self->is_initialised) 639 inflateEnd(&self->zst); 640 Dealloc(self); 641 } 642 643 /*[clinic input] 644 zlib.Compress.compress 645 646 data: Py_buffer 647 Binary data to be compressed. 648 / 649 650 Returns a bytes object containing compressed data. 651 652 After calling this function, some of the input data may still 653 be stored in internal buffers for later processing. 654 Call the flush() method to clear these buffers. 655 [clinic start generated code]*/ 656 657 static PyObject * 658 zlib_Compress_compress_impl(compobject *self, Py_buffer *data) 659 /*[clinic end generated code: output=5d5cd791cbc6a7f4 input=0d95908d6e64fab8]*/ 660 { 661 PyObject *RetVal = NULL; 662 Py_ssize_t ibuflen, obuflen = DEF_BUF_SIZE; 663 int err; 664 665 self->zst.next_in = data->buf; 666 ibuflen = data->len; 667 668 ENTER_ZLIB(self); 669 670 do { 671 arrange_input_buffer(&self->zst, &ibuflen); 672 673 do { 674 obuflen = arrange_output_buffer(&self->zst, &RetVal, obuflen); 675 if (obuflen < 0) 676 goto error; 677 678 Py_BEGIN_ALLOW_THREADS 679 err = deflate(&self->zst, Z_NO_FLUSH); 680 Py_END_ALLOW_THREADS 681 682 if (err == Z_STREAM_ERROR) { 683 zlib_error(self->zst, err, "while compressing data"); 684 goto error; 685 } 686 687 } while (self->zst.avail_out == 0); 688 assert(self->zst.avail_in == 0); 689 690 } while (ibuflen != 0); 691 692 if (_PyBytes_Resize(&RetVal, self->zst.next_out - 693 (Byte *)PyBytes_AS_STRING(RetVal)) == 0) 694 goto success; 695 696 error: 697 Py_CLEAR(RetVal); 698 success: 699 LEAVE_ZLIB(self); 700 return RetVal; 701 } 702 703 /* Helper for objdecompress() and flush(). Saves any unconsumed input data in 704 self->unused_data or self->unconsumed_tail, as appropriate. */ 705 static int 706 save_unconsumed_input(compobject *self, Py_buffer *data, int err) 707 { 708 if (err == Z_STREAM_END) { 709 /* The end of the compressed data has been reached. Store the leftover 710 input data in self->unused_data. */ 711 if (self->zst.avail_in > 0) { 712 Py_ssize_t old_size = PyBytes_GET_SIZE(self->unused_data); 713 Py_ssize_t new_size, left_size; 714 PyObject *new_data; 715 left_size = (Byte *)data->buf + data->len - self->zst.next_in; 716 if (left_size > (PY_SSIZE_T_MAX - old_size)) { 717 PyErr_NoMemory(); 718 return -1; 719 } 720 new_size = old_size + left_size; 721 new_data = PyBytes_FromStringAndSize(NULL, new_size); 722 if (new_data == NULL) 723 return -1; 724 memcpy(PyBytes_AS_STRING(new_data), 725 PyBytes_AS_STRING(self->unused_data), old_size); 726 memcpy(PyBytes_AS_STRING(new_data) + old_size, 727 self->zst.next_in, left_size); 728 Py_SETREF(self->unused_data, new_data); 729 self->zst.avail_in = 0; 730 } 731 } 732 733 if (self->zst.avail_in > 0 || PyBytes_GET_SIZE(self->unconsumed_tail)) { 734 /* This code handles two distinct cases: 735 1. Output limit was reached. Save leftover input in unconsumed_tail. 736 2. All input data was consumed. Clear unconsumed_tail. */ 737 Py_ssize_t left_size = (Byte *)data->buf + data->len - self->zst.next_in; 738 PyObject *new_data = PyBytes_FromStringAndSize( 739 (char *)self->zst.next_in, left_size); 740 if (new_data == NULL) 741 return -1; 742 Py_SETREF(self->unconsumed_tail, new_data); 743 } 744 745 return 0; 746 } 747 748 /*[clinic input] 749 zlib.Decompress.decompress 750 751 data: Py_buffer 752 The binary data to decompress. 753 / 754 max_length: ssize_t = 0 755 The maximum allowable length of the decompressed data. 756 Unconsumed input data will be stored in 757 the unconsumed_tail attribute. 758 759 Return a bytes object containing the decompressed version of the data. 760 761 After calling this function, some of the input data may still be stored in 762 internal buffers for later processing. 763 Call the flush() method to clear these buffers. 764 [clinic start generated code]*/ 765 766 static PyObject * 767 zlib_Decompress_decompress_impl(compobject *self, Py_buffer *data, 768 Py_ssize_t max_length) 769 /*[clinic end generated code: output=6e5173c74e710352 input=b85a212a012b770a]*/ 770 { 771 int err = Z_OK; 772 Py_ssize_t ibuflen, obuflen = DEF_BUF_SIZE, hard_limit; 773 PyObject *RetVal = NULL; 774 775 if (max_length < 0) { 776 PyErr_SetString(PyExc_ValueError, "max_length must be non-negative"); 777 return NULL; 778 } else if (max_length == 0) 779 hard_limit = PY_SSIZE_T_MAX; 780 else 781 hard_limit = max_length; 782 783 self->zst.next_in = data->buf; 784 ibuflen = data->len; 785 786 /* limit amount of data allocated to max_length */ 787 if (max_length && obuflen > max_length) 788 obuflen = max_length; 789 790 ENTER_ZLIB(self); 791 792 do { 793 arrange_input_buffer(&self->zst, &ibuflen); 794 795 do { 796 obuflen = arrange_output_buffer_with_maximum(&self->zst, &RetVal, 797 obuflen, hard_limit); 798 if (obuflen == -2) { 799 if (max_length > 0) { 800 goto save; 801 } 802 PyErr_NoMemory(); 803 } 804 if (obuflen < 0) { 805 goto abort; 806 } 807 808 Py_BEGIN_ALLOW_THREADS 809 err = inflate(&self->zst, Z_SYNC_FLUSH); 810 Py_END_ALLOW_THREADS 811 812 switch (err) { 813 case Z_OK: /* fall through */ 814 case Z_BUF_ERROR: /* fall through */ 815 case Z_STREAM_END: 816 break; 817 default: 818 if (err == Z_NEED_DICT && self->zdict != NULL) { 819 if (set_inflate_zdict(self) < 0) 820 goto abort; 821 else 822 break; 823 } 824 goto save; 825 } 826 827 } while (self->zst.avail_out == 0 || err == Z_NEED_DICT); 828 829 } while (err != Z_STREAM_END && ibuflen != 0); 830 831 save: 832 if (save_unconsumed_input(self, data, err) < 0) 833 goto abort; 834 835 if (err == Z_STREAM_END) { 836 /* This is the logical place to call inflateEnd, but the old behaviour 837 of only calling it on flush() is preserved. */ 838 self->eof = 1; 839 } else if (err != Z_OK && err != Z_BUF_ERROR) { 840 /* We will only get Z_BUF_ERROR if the output buffer was full 841 but there wasn't more output when we tried again, so it is 842 not an error condition. 843 */ 844 zlib_error(self->zst, err, "while decompressing data"); 845 goto abort; 846 } 847 848 if (_PyBytes_Resize(&RetVal, self->zst.next_out - 849 (Byte *)PyBytes_AS_STRING(RetVal)) == 0) 850 goto success; 851 852 abort: 853 Py_CLEAR(RetVal); 854 success: 855 LEAVE_ZLIB(self); 856 return RetVal; 857 } 858 859 /*[clinic input] 860 zlib.Compress.flush 861 862 mode: int(c_default="Z_FINISH") = zlib.Z_FINISH 863 One of the constants Z_SYNC_FLUSH, Z_FULL_FLUSH, Z_FINISH. 864 If mode == Z_FINISH, the compressor object can no longer be 865 used after calling the flush() method. Otherwise, more data 866 can still be compressed. 867 / 868 869 Return a bytes object containing any remaining compressed data. 870 [clinic start generated code]*/ 871 872 static PyObject * 873 zlib_Compress_flush_impl(compobject *self, int mode) 874 /*[clinic end generated code: output=a203f4cefc9de727 input=73ed066794bd15bc]*/ 875 { 876 int err; 877 Py_ssize_t length = DEF_BUF_SIZE; 878 PyObject *RetVal = NULL; 879 880 /* Flushing with Z_NO_FLUSH is a no-op, so there's no point in 881 doing any work at all; just return an empty string. */ 882 if (mode == Z_NO_FLUSH) { 883 return PyBytes_FromStringAndSize(NULL, 0); 884 } 885 886 ENTER_ZLIB(self); 887 888 self->zst.avail_in = 0; 889 890 do { 891 length = arrange_output_buffer(&self->zst, &RetVal, length); 892 if (length < 0) { 893 Py_CLEAR(RetVal); 894 goto error; 895 } 896 897 Py_BEGIN_ALLOW_THREADS 898 err = deflate(&self->zst, mode); 899 Py_END_ALLOW_THREADS 900 901 if (err == Z_STREAM_ERROR) { 902 zlib_error(self->zst, err, "while flushing"); 903 Py_CLEAR(RetVal); 904 goto error; 905 } 906 } while (self->zst.avail_out == 0); 907 assert(self->zst.avail_in == 0); 908 909 /* If mode is Z_FINISH, we also have to call deflateEnd() to free 910 various data structures. Note we should only get Z_STREAM_END when 911 mode is Z_FINISH, but checking both for safety*/ 912 if (err == Z_STREAM_END && mode == Z_FINISH) { 913 err = deflateEnd(&self->zst); 914 if (err != Z_OK) { 915 zlib_error(self->zst, err, "while finishing compression"); 916 Py_CLEAR(RetVal); 917 goto error; 918 } 919 else 920 self->is_initialised = 0; 921 922 /* We will only get Z_BUF_ERROR if the output buffer was full 923 but there wasn't more output when we tried again, so it is 924 not an error condition. 925 */ 926 } else if (err != Z_OK && err != Z_BUF_ERROR) { 927 zlib_error(self->zst, err, "while flushing"); 928 Py_CLEAR(RetVal); 929 goto error; 930 } 931 932 if (_PyBytes_Resize(&RetVal, self->zst.next_out - 933 (Byte *)PyBytes_AS_STRING(RetVal)) < 0) 934 Py_CLEAR(RetVal); 935 936 error: 937 LEAVE_ZLIB(self); 938 return RetVal; 939 } 940 941 #ifdef HAVE_ZLIB_COPY 942 943 /*[clinic input] 944 zlib.Compress.copy 945 946 Return a copy of the compression object. 947 [clinic start generated code]*/ 948 949 static PyObject * 950 zlib_Compress_copy_impl(compobject *self) 951 /*[clinic end generated code: output=5144aa153c21e805 input=c656351f94b82718]*/ 952 { 953 compobject *retval = NULL; 954 int err; 955 956 retval = newcompobject(&Comptype); 957 if (!retval) return NULL; 958 959 /* Copy the zstream state 960 * We use ENTER_ZLIB / LEAVE_ZLIB to make this thread-safe 961 */ 962 ENTER_ZLIB(self); 963 err = deflateCopy(&retval->zst, &self->zst); 964 switch (err) { 965 case Z_OK: 966 break; 967 case Z_STREAM_ERROR: 968 PyErr_SetString(PyExc_ValueError, "Inconsistent stream state"); 969 goto error; 970 case Z_MEM_ERROR: 971 PyErr_SetString(PyExc_MemoryError, 972 "Can't allocate memory for compression object"); 973 goto error; 974 default: 975 zlib_error(self->zst, err, "while copying compression object"); 976 goto error; 977 } 978 Py_INCREF(self->unused_data); 979 Py_XSETREF(retval->unused_data, self->unused_data); 980 Py_INCREF(self->unconsumed_tail); 981 Py_XSETREF(retval->unconsumed_tail, self->unconsumed_tail); 982 Py_XINCREF(self->zdict); 983 Py_XSETREF(retval->zdict, self->zdict); 984 retval->eof = self->eof; 985 986 /* Mark it as being initialized */ 987 retval->is_initialised = 1; 988 989 LEAVE_ZLIB(self); 990 return (PyObject *)retval; 991 992 error: 993 LEAVE_ZLIB(self); 994 Py_XDECREF(retval); 995 return NULL; 996 } 997 998 /*[clinic input] 999 zlib.Decompress.copy 1000 1001 Return a copy of the decompression object. 1002 [clinic start generated code]*/ 1003 1004 static PyObject * 1005 zlib_Decompress_copy_impl(compobject *self) 1006 /*[clinic end generated code: output=02a883a2a510c8cc input=ba6c3e96712a596b]*/ 1007 { 1008 compobject *retval = NULL; 1009 int err; 1010 1011 retval = newcompobject(&Decomptype); 1012 if (!retval) return NULL; 1013 1014 /* Copy the zstream state 1015 * We use ENTER_ZLIB / LEAVE_ZLIB to make this thread-safe 1016 */ 1017 ENTER_ZLIB(self); 1018 err = inflateCopy(&retval->zst, &self->zst); 1019 switch (err) { 1020 case Z_OK: 1021 break; 1022 case Z_STREAM_ERROR: 1023 PyErr_SetString(PyExc_ValueError, "Inconsistent stream state"); 1024 goto error; 1025 case Z_MEM_ERROR: 1026 PyErr_SetString(PyExc_MemoryError, 1027 "Can't allocate memory for decompression object"); 1028 goto error; 1029 default: 1030 zlib_error(self->zst, err, "while copying decompression object"); 1031 goto error; 1032 } 1033 1034 Py_INCREF(self->unused_data); 1035 Py_XSETREF(retval->unused_data, self->unused_data); 1036 Py_INCREF(self->unconsumed_tail); 1037 Py_XSETREF(retval->unconsumed_tail, self->unconsumed_tail); 1038 Py_XINCREF(self->zdict); 1039 Py_XSETREF(retval->zdict, self->zdict); 1040 retval->eof = self->eof; 1041 1042 /* Mark it as being initialized */ 1043 retval->is_initialised = 1; 1044 1045 LEAVE_ZLIB(self); 1046 return (PyObject *)retval; 1047 1048 error: 1049 LEAVE_ZLIB(self); 1050 Py_XDECREF(retval); 1051 return NULL; 1052 } 1053 #endif 1054 1055 /*[clinic input] 1056 zlib.Decompress.flush 1057 1058 length: ssize_t(c_default="DEF_BUF_SIZE") = zlib.DEF_BUF_SIZE 1059 the initial size of the output buffer. 1060 / 1061 1062 Return a bytes object containing any remaining decompressed data. 1063 [clinic start generated code]*/ 1064 1065 static PyObject * 1066 zlib_Decompress_flush_impl(compobject *self, Py_ssize_t length) 1067 /*[clinic end generated code: output=68c75ea127cbe654 input=aa4ec37f3aef4da0]*/ 1068 { 1069 int err, flush; 1070 Py_buffer data; 1071 PyObject *RetVal = NULL; 1072 Py_ssize_t ibuflen; 1073 1074 if (length <= 0) { 1075 PyErr_SetString(PyExc_ValueError, "length must be greater than zero"); 1076 return NULL; 1077 } 1078 1079 if (PyObject_GetBuffer(self->unconsumed_tail, &data, PyBUF_SIMPLE) == -1) 1080 return NULL; 1081 1082 ENTER_ZLIB(self); 1083 1084 self->zst.next_in = data.buf; 1085 ibuflen = data.len; 1086 1087 do { 1088 arrange_input_buffer(&self->zst, &ibuflen); 1089 flush = ibuflen == 0 ? Z_FINISH : Z_NO_FLUSH; 1090 1091 do { 1092 length = arrange_output_buffer(&self->zst, &RetVal, length); 1093 if (length < 0) 1094 goto abort; 1095 1096 Py_BEGIN_ALLOW_THREADS 1097 err = inflate(&self->zst, flush); 1098 Py_END_ALLOW_THREADS 1099 1100 switch (err) { 1101 case Z_OK: /* fall through */ 1102 case Z_BUF_ERROR: /* fall through */ 1103 case Z_STREAM_END: 1104 break; 1105 default: 1106 if (err == Z_NEED_DICT && self->zdict != NULL) { 1107 if (set_inflate_zdict(self) < 0) 1108 goto abort; 1109 else 1110 break; 1111 } 1112 goto save; 1113 } 1114 1115 } while (self->zst.avail_out == 0 || err == Z_NEED_DICT); 1116 1117 } while (err != Z_STREAM_END && ibuflen != 0); 1118 1119 save: 1120 if (save_unconsumed_input(self, &data, err) < 0) 1121 goto abort; 1122 1123 /* If at end of stream, clean up any memory allocated by zlib. */ 1124 if (err == Z_STREAM_END) { 1125 self->eof = 1; 1126 self->is_initialised = 0; 1127 err = inflateEnd(&self->zst); 1128 if (err != Z_OK) { 1129 zlib_error(self->zst, err, "while finishing decompression"); 1130 goto abort; 1131 } 1132 } 1133 1134 if (_PyBytes_Resize(&RetVal, self->zst.next_out - 1135 (Byte *)PyBytes_AS_STRING(RetVal)) == 0) 1136 goto success; 1137 1138 abort: 1139 Py_CLEAR(RetVal); 1140 success: 1141 PyBuffer_Release(&data); 1142 LEAVE_ZLIB(self); 1143 return RetVal; 1144 } 1145 1146 #include "clinic/zlibmodule.c.h" 1147 1148 static PyMethodDef comp_methods[] = 1149 { 1150 ZLIB_COMPRESS_COMPRESS_METHODDEF 1151 ZLIB_COMPRESS_FLUSH_METHODDEF 1152 #ifdef HAVE_ZLIB_COPY 1153 ZLIB_COMPRESS_COPY_METHODDEF 1154 #endif 1155 {NULL, NULL} 1156 }; 1157 1158 static PyMethodDef Decomp_methods[] = 1159 { 1160 ZLIB_DECOMPRESS_DECOMPRESS_METHODDEF 1161 ZLIB_DECOMPRESS_FLUSH_METHODDEF 1162 #ifdef HAVE_ZLIB_COPY 1163 ZLIB_DECOMPRESS_COPY_METHODDEF 1164 #endif 1165 {NULL, NULL} 1166 }; 1167 1168 #define COMP_OFF(x) offsetof(compobject, x) 1169 static PyMemberDef Decomp_members[] = { 1170 {"unused_data", T_OBJECT, COMP_OFF(unused_data), READONLY}, 1171 {"unconsumed_tail", T_OBJECT, COMP_OFF(unconsumed_tail), READONLY}, 1172 {"eof", T_BOOL, COMP_OFF(eof), READONLY}, 1173 {NULL}, 1174 }; 1175 1176 /*[clinic input] 1177 zlib.adler32 1178 1179 data: Py_buffer 1180 value: unsigned_int(bitwise=True) = 1 1181 Starting value of the checksum. 1182 / 1183 1184 Compute an Adler-32 checksum of data. 1185 1186 The returned checksum is an integer. 1187 [clinic start generated code]*/ 1188 1189 static PyObject * 1190 zlib_adler32_impl(PyObject *module, Py_buffer *data, unsigned int value) 1191 /*[clinic end generated code: output=422106f5ca8c92c0 input=6ff4557872160e88]*/ 1192 { 1193 /* Releasing the GIL for very small buffers is inefficient 1194 and may lower performance */ 1195 if (data->len > 1024*5) { 1196 unsigned char *buf = data->buf; 1197 Py_ssize_t len = data->len; 1198 1199 Py_BEGIN_ALLOW_THREADS 1200 /* Avoid truncation of length for very large buffers. adler32() takes 1201 length as an unsigned int, which may be narrower than Py_ssize_t. */ 1202 while ((size_t)len > UINT_MAX) { 1203 value = adler32(value, buf, UINT_MAX); 1204 buf += (size_t) UINT_MAX; 1205 len -= (size_t) UINT_MAX; 1206 } 1207 value = adler32(value, buf, (unsigned int)len); 1208 Py_END_ALLOW_THREADS 1209 } else { 1210 value = adler32(value, data->buf, (unsigned int)data->len); 1211 } 1212 return PyLong_FromUnsignedLong(value & 0xffffffffU); 1213 } 1214 1215 /*[clinic input] 1216 zlib.crc32 1217 1218 data: Py_buffer 1219 value: unsigned_int(bitwise=True) = 0 1220 Starting value of the checksum. 1221 / 1222 1223 Compute a CRC-32 checksum of data. 1224 1225 The returned checksum is an integer. 1226 [clinic start generated code]*/ 1227 1228 static PyObject * 1229 zlib_crc32_impl(PyObject *module, Py_buffer *data, unsigned int value) 1230 /*[clinic end generated code: output=63499fa20af7ea25 input=26c3ed430fa00b4c]*/ 1231 { 1232 int signed_val; 1233 1234 /* Releasing the GIL for very small buffers is inefficient 1235 and may lower performance */ 1236 if (data->len > 1024*5) { 1237 unsigned char *buf = data->buf; 1238 Py_ssize_t len = data->len; 1239 1240 Py_BEGIN_ALLOW_THREADS 1241 /* Avoid truncation of length for very large buffers. crc32() takes 1242 length as an unsigned int, which may be narrower than Py_ssize_t. */ 1243 while ((size_t)len > UINT_MAX) { 1244 value = crc32(value, buf, UINT_MAX); 1245 buf += (size_t) UINT_MAX; 1246 len -= (size_t) UINT_MAX; 1247 } 1248 signed_val = crc32(value, buf, (unsigned int)len); 1249 Py_END_ALLOW_THREADS 1250 } else { 1251 signed_val = crc32(value, data->buf, (unsigned int)data->len); 1252 } 1253 return PyLong_FromUnsignedLong(signed_val & 0xffffffffU); 1254 } 1255 1256 1257 static PyMethodDef zlib_methods[] = 1258 { 1259 ZLIB_ADLER32_METHODDEF 1260 ZLIB_COMPRESS_METHODDEF 1261 ZLIB_COMPRESSOBJ_METHODDEF 1262 ZLIB_CRC32_METHODDEF 1263 ZLIB_DECOMPRESS_METHODDEF 1264 ZLIB_DECOMPRESSOBJ_METHODDEF 1265 {NULL, NULL} 1266 }; 1267 1268 static PyTypeObject Comptype = { 1269 PyVarObject_HEAD_INIT(0, 0) 1270 "zlib.Compress", 1271 sizeof(compobject), 1272 0, 1273 (destructor)Comp_dealloc, /*tp_dealloc*/ 1274 0, /*tp_print*/ 1275 0, /*tp_getattr*/ 1276 0, /*tp_setattr*/ 1277 0, /*tp_reserved*/ 1278 0, /*tp_repr*/ 1279 0, /*tp_as_number*/ 1280 0, /*tp_as_sequence*/ 1281 0, /*tp_as_mapping*/ 1282 0, /*tp_hash*/ 1283 0, /*tp_call*/ 1284 0, /*tp_str*/ 1285 0, /*tp_getattro*/ 1286 0, /*tp_setattro*/ 1287 0, /*tp_as_buffer*/ 1288 Py_TPFLAGS_DEFAULT, /*tp_flags*/ 1289 0, /*tp_doc*/ 1290 0, /*tp_traverse*/ 1291 0, /*tp_clear*/ 1292 0, /*tp_richcompare*/ 1293 0, /*tp_weaklistoffset*/ 1294 0, /*tp_iter*/ 1295 0, /*tp_iternext*/ 1296 comp_methods, /*tp_methods*/ 1297 }; 1298 1299 static PyTypeObject Decomptype = { 1300 PyVarObject_HEAD_INIT(0, 0) 1301 "zlib.Decompress", 1302 sizeof(compobject), 1303 0, 1304 (destructor)Decomp_dealloc, /*tp_dealloc*/ 1305 0, /*tp_print*/ 1306 0, /*tp_getattr*/ 1307 0, /*tp_setattr*/ 1308 0, /*tp_reserved*/ 1309 0, /*tp_repr*/ 1310 0, /*tp_as_number*/ 1311 0, /*tp_as_sequence*/ 1312 0, /*tp_as_mapping*/ 1313 0, /*tp_hash*/ 1314 0, /*tp_call*/ 1315 0, /*tp_str*/ 1316 0, /*tp_getattro*/ 1317 0, /*tp_setattro*/ 1318 0, /*tp_as_buffer*/ 1319 Py_TPFLAGS_DEFAULT, /*tp_flags*/ 1320 0, /*tp_doc*/ 1321 0, /*tp_traverse*/ 1322 0, /*tp_clear*/ 1323 0, /*tp_richcompare*/ 1324 0, /*tp_weaklistoffset*/ 1325 0, /*tp_iter*/ 1326 0, /*tp_iternext*/ 1327 Decomp_methods, /*tp_methods*/ 1328 Decomp_members, /*tp_members*/ 1329 }; 1330 1331 PyDoc_STRVAR(zlib_module_documentation, 1332 "The functions in this module allow compression and decompression using the\n" 1333 "zlib library, which is based on GNU zip.\n" 1334 "\n" 1335 "adler32(string[, start]) -- Compute an Adler-32 checksum.\n" 1336 "compress(data[, level]) -- Compress data, with compression level 0-9 or -1.\n" 1337 "compressobj([level[, ...]]) -- Return a compressor object.\n" 1338 "crc32(string[, start]) -- Compute a CRC-32 checksum.\n" 1339 "decompress(string,[wbits],[bufsize]) -- Decompresses a compressed string.\n" 1340 "decompressobj([wbits[, zdict]]]) -- Return a decompressor object.\n" 1341 "\n" 1342 "'wbits' is window buffer size and container format.\n" 1343 "Compressor objects support compress() and flush() methods; decompressor\n" 1344 "objects support decompress() and flush()."); 1345 1346 static struct PyModuleDef zlibmodule = { 1347 PyModuleDef_HEAD_INIT, 1348 "zlib", 1349 zlib_module_documentation, 1350 -1, 1351 zlib_methods, 1352 NULL, 1353 NULL, 1354 NULL, 1355 NULL 1356 }; 1357 1358 PyMODINIT_FUNC 1359 PyInit_zlib(void) 1360 { 1361 PyObject *m, *ver; 1362 if (PyType_Ready(&Comptype) < 0) 1363 return NULL; 1364 if (PyType_Ready(&Decomptype) < 0) 1365 return NULL; 1366 m = PyModule_Create(&zlibmodule); 1367 if (m == NULL) 1368 return NULL; 1369 1370 ZlibError = PyErr_NewException("zlib.error", NULL, NULL); 1371 if (ZlibError != NULL) { 1372 Py_INCREF(ZlibError); 1373 PyModule_AddObject(m, "error", ZlibError); 1374 } 1375 PyModule_AddIntMacro(m, MAX_WBITS); 1376 PyModule_AddIntMacro(m, DEFLATED); 1377 PyModule_AddIntMacro(m, DEF_MEM_LEVEL); 1378 PyModule_AddIntMacro(m, DEF_BUF_SIZE); 1379 PyModule_AddIntMacro(m, Z_BEST_SPEED); 1380 PyModule_AddIntMacro(m, Z_BEST_COMPRESSION); 1381 PyModule_AddIntMacro(m, Z_DEFAULT_COMPRESSION); 1382 PyModule_AddIntMacro(m, Z_FILTERED); 1383 PyModule_AddIntMacro(m, Z_HUFFMAN_ONLY); 1384 PyModule_AddIntMacro(m, Z_DEFAULT_STRATEGY); 1385 1386 PyModule_AddIntMacro(m, Z_FINISH); 1387 PyModule_AddIntMacro(m, Z_NO_FLUSH); 1388 PyModule_AddIntMacro(m, Z_SYNC_FLUSH); 1389 PyModule_AddIntMacro(m, Z_FULL_FLUSH); 1390 1391 ver = PyUnicode_FromString(ZLIB_VERSION); 1392 if (ver != NULL) 1393 PyModule_AddObject(m, "ZLIB_VERSION", ver); 1394 1395 ver = PyUnicode_FromString(zlibVersion()); 1396 if (ver != NULL) 1397 PyModule_AddObject(m, "ZLIB_RUNTIME_VERSION", ver); 1398 1399 PyModule_AddStringConstant(m, "__version__", "1.0"); 1400 1401 return m; 1402 } 1403
