1 /* 2 * multibytecodec.c: Common Multibyte Codec Implementation 3 * 4 * Written by Hye-Shik Chang <perky (at) FreeBSD.org> 5 */ 6 7 #define PY_SSIZE_T_CLEAN 8 #include "Python.h" 9 #include "structmember.h" 10 #include "multibytecodec.h" 11 12 typedef struct { 13 const Py_UNICODE *inbuf, *inbuf_top, *inbuf_end; 14 unsigned char *outbuf, *outbuf_end; 15 PyObject *excobj, *outobj; 16 } MultibyteEncodeBuffer; 17 18 typedef struct { 19 const unsigned char *inbuf, *inbuf_top, *inbuf_end; 20 Py_UNICODE *outbuf, *outbuf_end; 21 PyObject *excobj, *outobj; 22 } MultibyteDecodeBuffer; 23 24 PyDoc_STRVAR(MultibyteCodec_Encode__doc__, 25 "I.encode(unicode[, errors]) -> (string, length consumed)\n\ 26 \n\ 27 Return an encoded string version of `unicode'. errors may be given to\n\ 28 set a different error handling scheme. Default is 'strict' meaning that\n\ 29 encoding errors raise a UnicodeEncodeError. Other possible values are\n\ 30 'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name\n\ 31 registered with codecs.register_error that can handle UnicodeEncodeErrors."); 32 33 PyDoc_STRVAR(MultibyteCodec_Decode__doc__, 34 "I.decode(string[, errors]) -> (unicodeobject, length consumed)\n\ 35 \n\ 36 Decodes `string' using I, an MultibyteCodec instance. errors may be given\n\ 37 to set a different error handling scheme. Default is 'strict' meaning\n\ 38 that encoding errors raise a UnicodeDecodeError. Other possible values\n\ 39 are 'ignore' and 'replace' as well as any other name registered with\n\ 40 codecs.register_error that is able to handle UnicodeDecodeErrors."); 41 42 static char *codeckwarglist[] = {"input", "errors", NULL}; 43 static char *incnewkwarglist[] = {"errors", NULL}; 44 static char *incrementalkwarglist[] = {"input", "final", NULL}; 45 static char *streamkwarglist[] = {"stream", "errors", NULL}; 46 47 static PyObject *multibytecodec_encode(MultibyteCodec *, 48 MultibyteCodec_State *, const Py_UNICODE **, Py_ssize_t, 49 PyObject *, int); 50 51 #define MBENC_RESET MBENC_MAX<<1 /* reset after an encoding session */ 52 53 static PyObject * 54 make_tuple(PyObject *object, Py_ssize_t len) 55 { 56 PyObject *v, *w; 57 58 if (object == NULL) 59 return NULL; 60 61 v = PyTuple_New(2); 62 if (v == NULL) { 63 Py_DECREF(object); 64 return NULL; 65 } 66 PyTuple_SET_ITEM(v, 0, object); 67 68 w = PyInt_FromSsize_t(len); 69 if (w == NULL) { 70 Py_DECREF(v); 71 return NULL; 72 } 73 PyTuple_SET_ITEM(v, 1, w); 74 75 return v; 76 } 77 78 static PyObject * 79 internal_error_callback(const char *errors) 80 { 81 if (errors == NULL || strcmp(errors, "strict") == 0) 82 return ERROR_STRICT; 83 else if (strcmp(errors, "ignore") == 0) 84 return ERROR_IGNORE; 85 else if (strcmp(errors, "replace") == 0) 86 return ERROR_REPLACE; 87 else 88 return PyString_FromString(errors); 89 } 90 91 static PyObject * 92 call_error_callback(PyObject *errors, PyObject *exc) 93 { 94 PyObject *args, *cb, *r; 95 96 assert(PyString_Check(errors)); 97 cb = PyCodec_LookupError(PyString_AS_STRING(errors)); 98 if (cb == NULL) 99 return NULL; 100 101 args = PyTuple_New(1); 102 if (args == NULL) { 103 Py_DECREF(cb); 104 return NULL; 105 } 106 107 PyTuple_SET_ITEM(args, 0, exc); 108 Py_INCREF(exc); 109 110 r = PyObject_CallObject(cb, args); 111 Py_DECREF(args); 112 Py_DECREF(cb); 113 return r; 114 } 115 116 static PyObject * 117 codecctx_errors_get(MultibyteStatefulCodecContext *self) 118 { 119 const char *errors; 120 121 if (self->errors == ERROR_STRICT) 122 errors = "strict"; 123 else if (self->errors == ERROR_IGNORE) 124 errors = "ignore"; 125 else if (self->errors == ERROR_REPLACE) 126 errors = "replace"; 127 else { 128 Py_INCREF(self->errors); 129 return self->errors; 130 } 131 132 return PyString_FromString(errors); 133 } 134 135 static int 136 codecctx_errors_set(MultibyteStatefulCodecContext *self, PyObject *value, 137 void *closure) 138 { 139 PyObject *cb; 140 141 if (!PyString_Check(value)) { 142 PyErr_SetString(PyExc_TypeError, "errors must be a string"); 143 return -1; 144 } 145 146 cb = internal_error_callback(PyString_AS_STRING(value)); 147 if (cb == NULL) 148 return -1; 149 150 ERROR_DECREF(self->errors); 151 self->errors = cb; 152 return 0; 153 } 154 155 /* This getset handlers list is used by all the stateful codec objects */ 156 static PyGetSetDef codecctx_getsets[] = { 157 {"errors", (getter)codecctx_errors_get, 158 (setter)codecctx_errors_set, 159 PyDoc_STR("how to treat errors")}, 160 {NULL,} 161 }; 162 163 static int 164 expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize) 165 { 166 Py_ssize_t orgpos, orgsize, incsize; 167 168 orgpos = (Py_ssize_t)((char *)buf->outbuf - 169 PyString_AS_STRING(buf->outobj)); 170 orgsize = PyString_GET_SIZE(buf->outobj); 171 incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize); 172 173 if (orgsize > PY_SSIZE_T_MAX - incsize) { 174 PyErr_NoMemory(); 175 return -1; 176 } 177 178 if (_PyString_Resize(&buf->outobj, orgsize + incsize) == -1) 179 return -1; 180 181 buf->outbuf = (unsigned char *)PyString_AS_STRING(buf->outobj) +orgpos; 182 buf->outbuf_end = (unsigned char *)PyString_AS_STRING(buf->outobj) 183 + PyString_GET_SIZE(buf->outobj); 184 185 return 0; 186 } 187 #define REQUIRE_ENCODEBUFFER(buf, s) do { \ 188 if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \ 189 if (expand_encodebuffer(buf, s) == -1) \ 190 goto errorexit; \ 191 } while(0) 192 193 static int 194 expand_decodebuffer(MultibyteDecodeBuffer *buf, Py_ssize_t esize) 195 { 196 Py_ssize_t orgpos, orgsize; 197 198 orgpos = (Py_ssize_t)(buf->outbuf - PyUnicode_AS_UNICODE(buf->outobj)); 199 orgsize = PyUnicode_GET_SIZE(buf->outobj); 200 if (PyUnicode_Resize(&buf->outobj, orgsize + ( 201 esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize)) == -1) 202 return -1; 203 204 buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj) + orgpos; 205 buf->outbuf_end = PyUnicode_AS_UNICODE(buf->outobj) 206 + PyUnicode_GET_SIZE(buf->outobj); 207 208 return 0; 209 } 210 #define REQUIRE_DECODEBUFFER(buf, s) do { \ 211 if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \ 212 if (expand_decodebuffer(buf, s) == -1) \ 213 goto errorexit; \ 214 } while(0) 215 216 217 /** 218 * MultibyteCodec object 219 */ 220 221 static int 222 multibytecodec_encerror(MultibyteCodec *codec, 223 MultibyteCodec_State *state, 224 MultibyteEncodeBuffer *buf, 225 PyObject *errors, Py_ssize_t e) 226 { 227 PyObject *retobj = NULL, *retstr = NULL, *tobj; 228 Py_ssize_t retstrsize, newpos; 229 Py_ssize_t esize, start, end; 230 const char *reason; 231 232 if (e > 0) { 233 reason = "illegal multibyte sequence"; 234 esize = e; 235 } 236 else { 237 switch (e) { 238 case MBERR_TOOSMALL: 239 REQUIRE_ENCODEBUFFER(buf, -1); 240 return 0; /* retry it */ 241 case MBERR_TOOFEW: 242 reason = "incomplete multibyte sequence"; 243 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); 244 break; 245 case MBERR_INTERNAL: 246 PyErr_SetString(PyExc_RuntimeError, 247 "internal codec error"); 248 return -1; 249 default: 250 PyErr_SetString(PyExc_RuntimeError, 251 "unknown runtime error"); 252 return -1; 253 } 254 } 255 256 if (errors == ERROR_REPLACE) { 257 const Py_UNICODE replchar = '?', *inbuf = &replchar; 258 Py_ssize_t r; 259 260 for (;;) { 261 Py_ssize_t outleft; 262 263 outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf); 264 r = codec->encode(state, codec->config, &inbuf, 1, 265 &buf->outbuf, outleft, 0); 266 if (r == MBERR_TOOSMALL) { 267 REQUIRE_ENCODEBUFFER(buf, -1); 268 continue; 269 } 270 else 271 break; 272 } 273 274 if (r != 0) { 275 REQUIRE_ENCODEBUFFER(buf, 1); 276 *buf->outbuf++ = '?'; 277 } 278 } 279 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) { 280 buf->inbuf += esize; 281 return 0; 282 } 283 284 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top); 285 end = start + esize; 286 287 /* use cached exception object if available */ 288 if (buf->excobj == NULL) { 289 buf->excobj = PyUnicodeEncodeError_Create(codec->encoding, 290 buf->inbuf_top, 291 buf->inbuf_end - buf->inbuf_top, 292 start, end, reason); 293 if (buf->excobj == NULL) 294 goto errorexit; 295 } 296 else 297 if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 || 298 PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 || 299 PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0) 300 goto errorexit; 301 302 if (errors == ERROR_STRICT) { 303 PyCodec_StrictErrors(buf->excobj); 304 goto errorexit; 305 } 306 307 retobj = call_error_callback(errors, buf->excobj); 308 if (retobj == NULL) 309 goto errorexit; 310 311 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 || 312 !PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) || 313 !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) || 314 PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) { 315 PyErr_SetString(PyExc_TypeError, 316 "encoding error handler must return " 317 "(unicode, int) tuple"); 318 goto errorexit; 319 } 320 321 { 322 const Py_UNICODE *uraw = PyUnicode_AS_UNICODE(tobj); 323 324 retstr = multibytecodec_encode(codec, state, &uraw, 325 PyUnicode_GET_SIZE(tobj), ERROR_STRICT, 326 MBENC_FLUSH); 327 if (retstr == NULL) 328 goto errorexit; 329 } 330 331 retstrsize = PyString_GET_SIZE(retstr); 332 if (retstrsize > 0) { 333 REQUIRE_ENCODEBUFFER(buf, retstrsize); 334 memcpy(buf->outbuf, PyString_AS_STRING(retstr), retstrsize); 335 buf->outbuf += retstrsize; 336 } 337 338 newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1)); 339 if (newpos < 0 && !PyErr_Occurred()) 340 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top); 341 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) { 342 PyErr_Clear(); 343 PyErr_Format(PyExc_IndexError, 344 "position %zd from error handler out of bounds", 345 newpos); 346 goto errorexit; 347 } 348 buf->inbuf = buf->inbuf_top + newpos; 349 350 Py_DECREF(retobj); 351 Py_DECREF(retstr); 352 return 0; 353 354 errorexit: 355 Py_XDECREF(retobj); 356 Py_XDECREF(retstr); 357 return -1; 358 } 359 360 static int 361 multibytecodec_decerror(MultibyteCodec *codec, 362 MultibyteCodec_State *state, 363 MultibyteDecodeBuffer *buf, 364 PyObject *errors, Py_ssize_t e) 365 { 366 PyObject *retobj = NULL, *retuni = NULL; 367 Py_ssize_t retunisize, newpos; 368 const char *reason; 369 Py_ssize_t esize, start, end; 370 371 if (e > 0) { 372 reason = "illegal multibyte sequence"; 373 esize = e; 374 } 375 else { 376 switch (e) { 377 case MBERR_TOOSMALL: 378 REQUIRE_DECODEBUFFER(buf, -1); 379 return 0; /* retry it */ 380 case MBERR_TOOFEW: 381 reason = "incomplete multibyte sequence"; 382 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); 383 break; 384 case MBERR_INTERNAL: 385 PyErr_SetString(PyExc_RuntimeError, 386 "internal codec error"); 387 return -1; 388 default: 389 PyErr_SetString(PyExc_RuntimeError, 390 "unknown runtime error"); 391 return -1; 392 } 393 } 394 395 if (errors == ERROR_REPLACE) { 396 REQUIRE_DECODEBUFFER(buf, 1); 397 *buf->outbuf++ = Py_UNICODE_REPLACEMENT_CHARACTER; 398 } 399 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) { 400 buf->inbuf += esize; 401 return 0; 402 } 403 404 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top); 405 end = start + esize; 406 407 /* use cached exception object if available */ 408 if (buf->excobj == NULL) { 409 buf->excobj = PyUnicodeDecodeError_Create(codec->encoding, 410 (const char *)buf->inbuf_top, 411 (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top), 412 start, end, reason); 413 if (buf->excobj == NULL) 414 goto errorexit; 415 } 416 else 417 if (PyUnicodeDecodeError_SetStart(buf->excobj, start) || 418 PyUnicodeDecodeError_SetEnd(buf->excobj, end) || 419 PyUnicodeDecodeError_SetReason(buf->excobj, reason)) 420 goto errorexit; 421 422 if (errors == ERROR_STRICT) { 423 PyCodec_StrictErrors(buf->excobj); 424 goto errorexit; 425 } 426 427 retobj = call_error_callback(errors, buf->excobj); 428 if (retobj == NULL) 429 goto errorexit; 430 431 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 || 432 !PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) || 433 !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) || 434 PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) { 435 PyErr_SetString(PyExc_TypeError, 436 "decoding error handler must return " 437 "(unicode, int) tuple"); 438 goto errorexit; 439 } 440 441 retunisize = PyUnicode_GET_SIZE(retuni); 442 if (retunisize > 0) { 443 REQUIRE_DECODEBUFFER(buf, retunisize); 444 memcpy((char *)buf->outbuf, PyUnicode_AS_DATA(retuni), 445 retunisize * Py_UNICODE_SIZE); 446 buf->outbuf += retunisize; 447 } 448 449 newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1)); 450 if (newpos < 0 && !PyErr_Occurred()) 451 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top); 452 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) { 453 PyErr_Clear(); 454 PyErr_Format(PyExc_IndexError, 455 "position %zd from error handler out of bounds", 456 newpos); 457 goto errorexit; 458 } 459 buf->inbuf = buf->inbuf_top + newpos; 460 Py_DECREF(retobj); 461 return 0; 462 463 errorexit: 464 Py_XDECREF(retobj); 465 return -1; 466 } 467 468 static PyObject * 469 multibytecodec_encode(MultibyteCodec *codec, 470 MultibyteCodec_State *state, 471 const Py_UNICODE **data, Py_ssize_t datalen, 472 PyObject *errors, int flags) 473 { 474 MultibyteEncodeBuffer buf; 475 Py_ssize_t finalsize, r = 0; 476 477 if (datalen == 0 && !(flags & MBENC_RESET)) 478 return PyString_FromString(""); 479 480 buf.excobj = NULL; 481 buf.inbuf = buf.inbuf_top = *data; 482 buf.inbuf_end = buf.inbuf_top + datalen; 483 484 if (datalen > (PY_SSIZE_T_MAX - 16) / 2) { 485 PyErr_NoMemory(); 486 goto errorexit; 487 } 488 489 buf.outobj = PyString_FromStringAndSize(NULL, datalen * 2 + 16); 490 if (buf.outobj == NULL) 491 goto errorexit; 492 buf.outbuf = (unsigned char *)PyString_AS_STRING(buf.outobj); 493 buf.outbuf_end = buf.outbuf + PyString_GET_SIZE(buf.outobj); 494 495 while (buf.inbuf < buf.inbuf_end) { 496 Py_ssize_t inleft, outleft; 497 498 /* we don't reuse inleft and outleft here. 499 * error callbacks can relocate the cursor anywhere on buffer*/ 500 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf); 501 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf); 502 r = codec->encode(state, codec->config, &buf.inbuf, inleft, 503 &buf.outbuf, outleft, flags); 504 if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH))) 505 break; 506 else if (multibytecodec_encerror(codec, state, &buf, errors,r)) 507 goto errorexit; 508 else if (r == MBERR_TOOFEW) 509 break; 510 } 511 512 if (codec->encreset != NULL && (flags & MBENC_RESET)) 513 for (;;) { 514 Py_ssize_t outleft; 515 516 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf); 517 r = codec->encreset(state, codec->config, &buf.outbuf, 518 outleft); 519 if (r == 0) 520 break; 521 else if (multibytecodec_encerror(codec, state, 522 &buf, errors, r)) 523 goto errorexit; 524 } 525 526 finalsize = (Py_ssize_t)((char *)buf.outbuf - 527 PyString_AS_STRING(buf.outobj)); 528 529 if (finalsize != PyString_GET_SIZE(buf.outobj)) 530 if (_PyString_Resize(&buf.outobj, finalsize) == -1) 531 goto errorexit; 532 533 *data = buf.inbuf; 534 Py_XDECREF(buf.excobj); 535 return buf.outobj; 536 537 errorexit: 538 Py_XDECREF(buf.excobj); 539 Py_XDECREF(buf.outobj); 540 return NULL; 541 } 542 543 static PyObject * 544 MultibyteCodec_Encode(MultibyteCodecObject *self, 545 PyObject *args, PyObject *kwargs) 546 { 547 MultibyteCodec_State state; 548 Py_UNICODE *data; 549 PyObject *errorcb, *r, *arg, *ucvt; 550 const char *errors = NULL; 551 Py_ssize_t datalen; 552 553 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|z:encode", 554 codeckwarglist, &arg, &errors)) 555 return NULL; 556 557 if (PyUnicode_Check(arg)) 558 ucvt = NULL; 559 else { 560 arg = ucvt = PyObject_Unicode(arg); 561 if (arg == NULL) 562 return NULL; 563 else if (!PyUnicode_Check(arg)) { 564 PyErr_SetString(PyExc_TypeError, 565 "couldn't convert the object to unicode."); 566 Py_DECREF(ucvt); 567 return NULL; 568 } 569 } 570 571 data = PyUnicode_AS_UNICODE(arg); 572 datalen = PyUnicode_GET_SIZE(arg); 573 574 errorcb = internal_error_callback(errors); 575 if (errorcb == NULL) { 576 Py_XDECREF(ucvt); 577 return NULL; 578 } 579 580 if (self->codec->encinit != NULL && 581 self->codec->encinit(&state, self->codec->config) != 0) 582 goto errorexit; 583 r = multibytecodec_encode(self->codec, &state, 584 (const Py_UNICODE **)&data, datalen, errorcb, 585 MBENC_FLUSH | MBENC_RESET); 586 if (r == NULL) 587 goto errorexit; 588 589 ERROR_DECREF(errorcb); 590 Py_XDECREF(ucvt); 591 return make_tuple(r, datalen); 592 593 errorexit: 594 ERROR_DECREF(errorcb); 595 Py_XDECREF(ucvt); 596 return NULL; 597 } 598 599 static PyObject * 600 MultibyteCodec_Decode(MultibyteCodecObject *self, 601 PyObject *args, PyObject *kwargs) 602 { 603 MultibyteCodec_State state; 604 MultibyteDecodeBuffer buf; 605 PyObject *errorcb; 606 Py_buffer pdata; 607 const char *data, *errors = NULL; 608 Py_ssize_t datalen, finalsize; 609 610 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|z:decode", 611 codeckwarglist, &pdata, &errors)) 612 return NULL; 613 data = pdata.buf; 614 datalen = pdata.len; 615 616 errorcb = internal_error_callback(errors); 617 if (errorcb == NULL) { 618 PyBuffer_Release(&pdata); 619 return NULL; 620 } 621 622 if (datalen == 0) { 623 PyBuffer_Release(&pdata); 624 ERROR_DECREF(errorcb); 625 return make_tuple(PyUnicode_FromUnicode(NULL, 0), 0); 626 } 627 628 buf.excobj = NULL; 629 buf.inbuf = buf.inbuf_top = (unsigned char *)data; 630 buf.inbuf_end = buf.inbuf_top + datalen; 631 buf.outobj = PyUnicode_FromUnicode(NULL, datalen); 632 if (buf.outobj == NULL) 633 goto errorexit; 634 buf.outbuf = PyUnicode_AS_UNICODE(buf.outobj); 635 buf.outbuf_end = buf.outbuf + PyUnicode_GET_SIZE(buf.outobj); 636 637 if (self->codec->decinit != NULL && 638 self->codec->decinit(&state, self->codec->config) != 0) 639 goto errorexit; 640 641 while (buf.inbuf < buf.inbuf_end) { 642 Py_ssize_t inleft, outleft, r; 643 644 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf); 645 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf); 646 647 r = self->codec->decode(&state, self->codec->config, 648 &buf.inbuf, inleft, &buf.outbuf, outleft); 649 if (r == 0) 650 break; 651 else if (multibytecodec_decerror(self->codec, &state, 652 &buf, errorcb, r)) 653 goto errorexit; 654 } 655 656 finalsize = (Py_ssize_t)(buf.outbuf - 657 PyUnicode_AS_UNICODE(buf.outobj)); 658 659 if (finalsize != PyUnicode_GET_SIZE(buf.outobj)) 660 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1) 661 goto errorexit; 662 663 PyBuffer_Release(&pdata); 664 Py_XDECREF(buf.excobj); 665 ERROR_DECREF(errorcb); 666 return make_tuple(buf.outobj, datalen); 667 668 errorexit: 669 PyBuffer_Release(&pdata); 670 ERROR_DECREF(errorcb); 671 Py_XDECREF(buf.excobj); 672 Py_XDECREF(buf.outobj); 673 674 return NULL; 675 } 676 677 static struct PyMethodDef multibytecodec_methods[] = { 678 {"encode", (PyCFunction)MultibyteCodec_Encode, 679 METH_VARARGS | METH_KEYWORDS, 680 MultibyteCodec_Encode__doc__}, 681 {"decode", (PyCFunction)MultibyteCodec_Decode, 682 METH_VARARGS | METH_KEYWORDS, 683 MultibyteCodec_Decode__doc__}, 684 {NULL, NULL}, 685 }; 686 687 static void 688 multibytecodec_dealloc(MultibyteCodecObject *self) 689 { 690 PyObject_Del(self); 691 } 692 693 static PyTypeObject MultibyteCodec_Type = { 694 PyVarObject_HEAD_INIT(NULL, 0) 695 "MultibyteCodec", /* tp_name */ 696 sizeof(MultibyteCodecObject), /* tp_basicsize */ 697 0, /* tp_itemsize */ 698 /* methods */ 699 (destructor)multibytecodec_dealloc, /* tp_dealloc */ 700 0, /* tp_print */ 701 0, /* tp_getattr */ 702 0, /* tp_setattr */ 703 0, /* tp_compare */ 704 0, /* tp_repr */ 705 0, /* tp_as_number */ 706 0, /* tp_as_sequence */ 707 0, /* tp_as_mapping */ 708 0, /* tp_hash */ 709 0, /* tp_call */ 710 0, /* tp_str */ 711 PyObject_GenericGetAttr, /* tp_getattro */ 712 0, /* tp_setattro */ 713 0, /* tp_as_buffer */ 714 Py_TPFLAGS_DEFAULT, /* tp_flags */ 715 0, /* tp_doc */ 716 0, /* tp_traverse */ 717 0, /* tp_clear */ 718 0, /* tp_richcompare */ 719 0, /* tp_weaklistoffset */ 720 0, /* tp_iter */ 721 0, /* tp_iterext */ 722 multibytecodec_methods, /* tp_methods */ 723 }; 724 725 726 /** 727 * Utility functions for stateful codec mechanism 728 */ 729 730 #define STATEFUL_DCTX(o) ((MultibyteStatefulDecoderContext *)(o)) 731 #define STATEFUL_ECTX(o) ((MultibyteStatefulEncoderContext *)(o)) 732 733 static PyObject * 734 encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx, 735 PyObject *unistr, int final) 736 { 737 PyObject *ucvt, *r = NULL; 738 Py_UNICODE *inbuf, *inbuf_end, *inbuf_tmp = NULL; 739 Py_ssize_t datalen, origpending; 740 741 if (PyUnicode_Check(unistr)) 742 ucvt = NULL; 743 else { 744 unistr = ucvt = PyObject_Unicode(unistr); 745 if (unistr == NULL) 746 return NULL; 747 else if (!PyUnicode_Check(unistr)) { 748 PyErr_SetString(PyExc_TypeError, 749 "couldn't convert the object to unicode."); 750 Py_DECREF(ucvt); 751 return NULL; 752 } 753 } 754 755 datalen = PyUnicode_GET_SIZE(unistr); 756 origpending = ctx->pendingsize; 757 758 if (origpending > 0) { 759 if (datalen > PY_SSIZE_T_MAX - ctx->pendingsize) { 760 PyErr_NoMemory(); 761 /* inbuf_tmp == NULL */ 762 goto errorexit; 763 } 764 inbuf_tmp = PyMem_New(Py_UNICODE, datalen + ctx->pendingsize); 765 if (inbuf_tmp == NULL) 766 goto errorexit; 767 memcpy(inbuf_tmp, ctx->pending, 768 Py_UNICODE_SIZE * ctx->pendingsize); 769 memcpy(inbuf_tmp + ctx->pendingsize, 770 PyUnicode_AS_UNICODE(unistr), 771 Py_UNICODE_SIZE * datalen); 772 datalen += ctx->pendingsize; 773 ctx->pendingsize = 0; 774 inbuf = inbuf_tmp; 775 } 776 else 777 inbuf = (Py_UNICODE *)PyUnicode_AS_UNICODE(unistr); 778 779 inbuf_end = inbuf + datalen; 780 781 r = multibytecodec_encode(ctx->codec, &ctx->state, 782 (const Py_UNICODE **)&inbuf, datalen, 783 ctx->errors, final ? MBENC_FLUSH | MBENC_RESET : 0); 784 if (r == NULL) { 785 /* recover the original pending buffer */ 786 if (origpending > 0) 787 memcpy(ctx->pending, inbuf_tmp, 788 Py_UNICODE_SIZE * origpending); 789 ctx->pendingsize = origpending; 790 goto errorexit; 791 } 792 793 if (inbuf < inbuf_end) { 794 ctx->pendingsize = (Py_ssize_t)(inbuf_end - inbuf); 795 if (ctx->pendingsize > MAXENCPENDING) { 796 /* normal codecs can't reach here */ 797 ctx->pendingsize = 0; 798 PyErr_SetString(PyExc_UnicodeError, 799 "pending buffer overflow"); 800 goto errorexit; 801 } 802 memcpy(ctx->pending, inbuf, 803 ctx->pendingsize * Py_UNICODE_SIZE); 804 } 805 806 if (inbuf_tmp != NULL) 807 PyMem_Del(inbuf_tmp); 808 Py_XDECREF(ucvt); 809 return r; 810 811 errorexit: 812 if (inbuf_tmp != NULL) 813 PyMem_Del(inbuf_tmp); 814 Py_XDECREF(r); 815 Py_XDECREF(ucvt); 816 return NULL; 817 } 818 819 static int 820 decoder_append_pending(MultibyteStatefulDecoderContext *ctx, 821 MultibyteDecodeBuffer *buf) 822 { 823 Py_ssize_t npendings; 824 825 npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); 826 if (npendings + ctx->pendingsize > MAXDECPENDING || 827 npendings > PY_SSIZE_T_MAX - ctx->pendingsize) { 828 PyErr_SetString(PyExc_UnicodeError, "pending buffer overflow"); 829 return -1; 830 } 831 memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings); 832 ctx->pendingsize += npendings; 833 return 0; 834 } 835 836 static int 837 decoder_prepare_buffer(MultibyteDecodeBuffer *buf, const char *data, 838 Py_ssize_t size) 839 { 840 buf->inbuf = buf->inbuf_top = (const unsigned char *)data; 841 buf->inbuf_end = buf->inbuf_top + size; 842 if (buf->outobj == NULL) { /* only if outobj is not allocated yet */ 843 buf->outobj = PyUnicode_FromUnicode(NULL, size); 844 if (buf->outobj == NULL) 845 return -1; 846 buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj); 847 buf->outbuf_end = buf->outbuf + 848 PyUnicode_GET_SIZE(buf->outobj); 849 } 850 851 return 0; 852 } 853 854 static int 855 decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx, 856 MultibyteDecodeBuffer *buf) 857 { 858 while (buf->inbuf < buf->inbuf_end) { 859 Py_ssize_t inleft, outleft; 860 Py_ssize_t r; 861 862 inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); 863 outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf); 864 865 r = ctx->codec->decode(&ctx->state, ctx->codec->config, 866 &buf->inbuf, inleft, &buf->outbuf, outleft); 867 if (r == 0 || r == MBERR_TOOFEW) 868 break; 869 else if (multibytecodec_decerror(ctx->codec, &ctx->state, 870 buf, ctx->errors, r)) 871 return -1; 872 } 873 return 0; 874 } 875 876 877 /** 878 * MultibyteIncrementalEncoder object 879 */ 880 881 static PyObject * 882 mbiencoder_encode(MultibyteIncrementalEncoderObject *self, 883 PyObject *args, PyObject *kwargs) 884 { 885 PyObject *data; 886 int final = 0; 887 888 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i:encode", 889 incrementalkwarglist, &data, &final)) 890 return NULL; 891 892 return encoder_encode_stateful(STATEFUL_ECTX(self), data, final); 893 } 894 895 static PyObject * 896 mbiencoder_reset(MultibyteIncrementalEncoderObject *self) 897 { 898 if (self->codec->decreset != NULL && 899 self->codec->decreset(&self->state, self->codec->config) != 0) 900 return NULL; 901 self->pendingsize = 0; 902 903 Py_RETURN_NONE; 904 } 905 906 static struct PyMethodDef mbiencoder_methods[] = { 907 {"encode", (PyCFunction)mbiencoder_encode, 908 METH_VARARGS | METH_KEYWORDS, NULL}, 909 {"reset", (PyCFunction)mbiencoder_reset, 910 METH_NOARGS, NULL}, 911 {NULL, NULL}, 912 }; 913 914 static PyObject * 915 mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) 916 { 917 MultibyteIncrementalEncoderObject *self; 918 PyObject *codec = NULL; 919 char *errors = NULL; 920 921 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalEncoder", 922 incnewkwarglist, &errors)) 923 return NULL; 924 925 self = (MultibyteIncrementalEncoderObject *)type->tp_alloc(type, 0); 926 if (self == NULL) 927 return NULL; 928 929 codec = PyObject_GetAttrString((PyObject *)type, "codec"); 930 if (codec == NULL) 931 goto errorexit; 932 if (!MultibyteCodec_Check(codec)) { 933 PyErr_SetString(PyExc_TypeError, "codec is unexpected type"); 934 goto errorexit; 935 } 936 937 self->codec = ((MultibyteCodecObject *)codec)->codec; 938 self->pendingsize = 0; 939 self->errors = internal_error_callback(errors); 940 if (self->errors == NULL) 941 goto errorexit; 942 if (self->codec->encinit != NULL && 943 self->codec->encinit(&self->state, self->codec->config) != 0) 944 goto errorexit; 945 946 Py_DECREF(codec); 947 return (PyObject *)self; 948 949 errorexit: 950 Py_XDECREF(self); 951 Py_XDECREF(codec); 952 return NULL; 953 } 954 955 static int 956 mbiencoder_init(PyObject *self, PyObject *args, PyObject *kwds) 957 { 958 return 0; 959 } 960 961 static int 962 mbiencoder_traverse(MultibyteIncrementalEncoderObject *self, 963 visitproc visit, void *arg) 964 { 965 if (ERROR_ISCUSTOM(self->errors)) 966 Py_VISIT(self->errors); 967 return 0; 968 } 969 970 static void 971 mbiencoder_dealloc(MultibyteIncrementalEncoderObject *self) 972 { 973 PyObject_GC_UnTrack(self); 974 ERROR_DECREF(self->errors); 975 Py_TYPE(self)->tp_free(self); 976 } 977 978 static PyTypeObject MultibyteIncrementalEncoder_Type = { 979 PyVarObject_HEAD_INIT(NULL, 0) 980 "MultibyteIncrementalEncoder", /* tp_name */ 981 sizeof(MultibyteIncrementalEncoderObject), /* tp_basicsize */ 982 0, /* tp_itemsize */ 983 /* methods */ 984 (destructor)mbiencoder_dealloc, /* tp_dealloc */ 985 0, /* tp_print */ 986 0, /* tp_getattr */ 987 0, /* tp_setattr */ 988 0, /* tp_compare */ 989 0, /* tp_repr */ 990 0, /* tp_as_number */ 991 0, /* tp_as_sequence */ 992 0, /* tp_as_mapping */ 993 0, /* tp_hash */ 994 0, /* tp_call */ 995 0, /* tp_str */ 996 PyObject_GenericGetAttr, /* tp_getattro */ 997 0, /* tp_setattro */ 998 0, /* tp_as_buffer */ 999 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC 1000 | Py_TPFLAGS_BASETYPE, /* tp_flags */ 1001 0, /* tp_doc */ 1002 (traverseproc)mbiencoder_traverse, /* tp_traverse */ 1003 0, /* tp_clear */ 1004 0, /* tp_richcompare */ 1005 0, /* tp_weaklistoffset */ 1006 0, /* tp_iter */ 1007 0, /* tp_iterext */ 1008 mbiencoder_methods, /* tp_methods */ 1009 0, /* tp_members */ 1010 codecctx_getsets, /* tp_getset */ 1011 0, /* tp_base */ 1012 0, /* tp_dict */ 1013 0, /* tp_descr_get */ 1014 0, /* tp_descr_set */ 1015 0, /* tp_dictoffset */ 1016 mbiencoder_init, /* tp_init */ 1017 0, /* tp_alloc */ 1018 mbiencoder_new, /* tp_new */ 1019 }; 1020 1021 1022 /** 1023 * MultibyteIncrementalDecoder object 1024 */ 1025 1026 static PyObject * 1027 mbidecoder_decode(MultibyteIncrementalDecoderObject *self, 1028 PyObject *args, PyObject *kwargs) 1029 { 1030 MultibyteDecodeBuffer buf; 1031 char *data, *wdata = NULL; 1032 Py_buffer pdata; 1033 Py_ssize_t wsize, finalsize = 0, size, origpending; 1034 int final = 0; 1035 1036 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|i:decode", 1037 incrementalkwarglist, &pdata, &final)) 1038 return NULL; 1039 data = pdata.buf; 1040 size = pdata.len; 1041 1042 buf.outobj = buf.excobj = NULL; 1043 origpending = self->pendingsize; 1044 1045 if (self->pendingsize == 0) { 1046 wsize = size; 1047 wdata = data; 1048 } 1049 else { 1050 if (size > PY_SSIZE_T_MAX - self->pendingsize) { 1051 PyErr_NoMemory(); 1052 goto errorexit; 1053 } 1054 wsize = size + self->pendingsize; 1055 wdata = PyMem_Malloc(wsize); 1056 if (wdata == NULL) 1057 goto errorexit; 1058 memcpy(wdata, self->pending, self->pendingsize); 1059 memcpy(wdata + self->pendingsize, data, size); 1060 self->pendingsize = 0; 1061 } 1062 1063 if (decoder_prepare_buffer(&buf, wdata, wsize) != 0) 1064 goto errorexit; 1065 1066 if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf)) 1067 goto errorexit; 1068 1069 if (final && buf.inbuf < buf.inbuf_end) { 1070 if (multibytecodec_decerror(self->codec, &self->state, 1071 &buf, self->errors, MBERR_TOOFEW)) { 1072 /* recover the original pending buffer */ 1073 memcpy(self->pending, wdata, origpending); 1074 self->pendingsize = origpending; 1075 goto errorexit; 1076 } 1077 } 1078 1079 if (buf.inbuf < buf.inbuf_end) { /* pending sequence still exists */ 1080 if (decoder_append_pending(STATEFUL_DCTX(self), &buf) != 0) 1081 goto errorexit; 1082 } 1083 1084 finalsize = (Py_ssize_t)(buf.outbuf - PyUnicode_AS_UNICODE(buf.outobj)); 1085 if (finalsize != PyUnicode_GET_SIZE(buf.outobj)) 1086 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1) 1087 goto errorexit; 1088 1089 PyBuffer_Release(&pdata); 1090 if (wdata != data) 1091 PyMem_Del(wdata); 1092 Py_XDECREF(buf.excobj); 1093 return buf.outobj; 1094 1095 errorexit: 1096 PyBuffer_Release(&pdata); 1097 if (wdata != NULL && wdata != data) 1098 PyMem_Del(wdata); 1099 Py_XDECREF(buf.excobj); 1100 Py_XDECREF(buf.outobj); 1101 return NULL; 1102 } 1103 1104 static PyObject * 1105 mbidecoder_reset(MultibyteIncrementalDecoderObject *self) 1106 { 1107 if (self->codec->decreset != NULL && 1108 self->codec->decreset(&self->state, self->codec->config) != 0) 1109 return NULL; 1110 self->pendingsize = 0; 1111 1112 Py_RETURN_NONE; 1113 } 1114 1115 static struct PyMethodDef mbidecoder_methods[] = { 1116 {"decode", (PyCFunction)mbidecoder_decode, 1117 METH_VARARGS | METH_KEYWORDS, NULL}, 1118 {"reset", (PyCFunction)mbidecoder_reset, 1119 METH_NOARGS, NULL}, 1120 {NULL, NULL}, 1121 }; 1122 1123 static PyObject * 1124 mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) 1125 { 1126 MultibyteIncrementalDecoderObject *self; 1127 PyObject *codec = NULL; 1128 char *errors = NULL; 1129 1130 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalDecoder", 1131 incnewkwarglist, &errors)) 1132 return NULL; 1133 1134 self = (MultibyteIncrementalDecoderObject *)type->tp_alloc(type, 0); 1135 if (self == NULL) 1136 return NULL; 1137 1138 codec = PyObject_GetAttrString((PyObject *)type, "codec"); 1139 if (codec == NULL) 1140 goto errorexit; 1141 if (!MultibyteCodec_Check(codec)) { 1142 PyErr_SetString(PyExc_TypeError, "codec is unexpected type"); 1143 goto errorexit; 1144 } 1145 1146 self->codec = ((MultibyteCodecObject *)codec)->codec; 1147 self->pendingsize = 0; 1148 self->errors = internal_error_callback(errors); 1149 if (self->errors == NULL) 1150 goto errorexit; 1151 if (self->codec->decinit != NULL && 1152 self->codec->decinit(&self->state, self->codec->config) != 0) 1153 goto errorexit; 1154 1155 Py_DECREF(codec); 1156 return (PyObject *)self; 1157 1158 errorexit: 1159 Py_XDECREF(self); 1160 Py_XDECREF(codec); 1161 return NULL; 1162 } 1163 1164 static int 1165 mbidecoder_init(PyObject *self, PyObject *args, PyObject *kwds) 1166 { 1167 return 0; 1168 } 1169 1170 static int 1171 mbidecoder_traverse(MultibyteIncrementalDecoderObject *self, 1172 visitproc visit, void *arg) 1173 { 1174 if (ERROR_ISCUSTOM(self->errors)) 1175 Py_VISIT(self->errors); 1176 return 0; 1177 } 1178 1179 static void 1180 mbidecoder_dealloc(MultibyteIncrementalDecoderObject *self) 1181 { 1182 PyObject_GC_UnTrack(self); 1183 ERROR_DECREF(self->errors); 1184 Py_TYPE(self)->tp_free(self); 1185 } 1186 1187 static PyTypeObject MultibyteIncrementalDecoder_Type = { 1188 PyVarObject_HEAD_INIT(NULL, 0) 1189 "MultibyteIncrementalDecoder", /* tp_name */ 1190 sizeof(MultibyteIncrementalDecoderObject), /* tp_basicsize */ 1191 0, /* tp_itemsize */ 1192 /* methods */ 1193 (destructor)mbidecoder_dealloc, /* tp_dealloc */ 1194 0, /* tp_print */ 1195 0, /* tp_getattr */ 1196 0, /* tp_setattr */ 1197 0, /* tp_compare */ 1198 0, /* tp_repr */ 1199 0, /* tp_as_number */ 1200 0, /* tp_as_sequence */ 1201 0, /* tp_as_mapping */ 1202 0, /* tp_hash */ 1203 0, /* tp_call */ 1204 0, /* tp_str */ 1205 PyObject_GenericGetAttr, /* tp_getattro */ 1206 0, /* tp_setattro */ 1207 0, /* tp_as_buffer */ 1208 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC 1209 | Py_TPFLAGS_BASETYPE, /* tp_flags */ 1210 0, /* tp_doc */ 1211 (traverseproc)mbidecoder_traverse, /* tp_traverse */ 1212 0, /* tp_clear */ 1213 0, /* tp_richcompare */ 1214 0, /* tp_weaklistoffset */ 1215 0, /* tp_iter */ 1216 0, /* tp_iterext */ 1217 mbidecoder_methods, /* tp_methods */ 1218 0, /* tp_members */ 1219 codecctx_getsets, /* tp_getset */ 1220 0, /* tp_base */ 1221 0, /* tp_dict */ 1222 0, /* tp_descr_get */ 1223 0, /* tp_descr_set */ 1224 0, /* tp_dictoffset */ 1225 mbidecoder_init, /* tp_init */ 1226 0, /* tp_alloc */ 1227 mbidecoder_new, /* tp_new */ 1228 }; 1229 1230 1231 /** 1232 * MultibyteStreamReader object 1233 */ 1234 1235 static PyObject * 1236 mbstreamreader_iread(MultibyteStreamReaderObject *self, 1237 const char *method, Py_ssize_t sizehint) 1238 { 1239 MultibyteDecodeBuffer buf; 1240 PyObject *cres; 1241 Py_ssize_t rsize, finalsize = 0; 1242 1243 if (sizehint == 0) 1244 return PyUnicode_FromUnicode(NULL, 0); 1245 1246 buf.outobj = buf.excobj = NULL; 1247 cres = NULL; 1248 1249 for (;;) { 1250 int endoffile; 1251 1252 if (sizehint < 0) 1253 cres = PyObject_CallMethod(self->stream, 1254 (char *)method, NULL); 1255 else 1256 cres = PyObject_CallMethod(self->stream, 1257 (char *)method, "i", sizehint); 1258 if (cres == NULL) 1259 goto errorexit; 1260 1261 if (!PyString_Check(cres)) { 1262 PyErr_SetString(PyExc_TypeError, 1263 "stream function returned a " 1264 "non-string object"); 1265 goto errorexit; 1266 } 1267 1268 endoffile = (PyString_GET_SIZE(cres) == 0); 1269 1270 if (self->pendingsize > 0) { 1271 PyObject *ctr; 1272 char *ctrdata; 1273 1274 if (PyString_GET_SIZE(cres) > PY_SSIZE_T_MAX - self->pendingsize) { 1275 PyErr_NoMemory(); 1276 goto errorexit; 1277 } 1278 rsize = PyString_GET_SIZE(cres) + self->pendingsize; 1279 ctr = PyString_FromStringAndSize(NULL, rsize); 1280 if (ctr == NULL) 1281 goto errorexit; 1282 ctrdata = PyString_AS_STRING(ctr); 1283 memcpy(ctrdata, self->pending, self->pendingsize); 1284 memcpy(ctrdata + self->pendingsize, 1285 PyString_AS_STRING(cres), 1286 PyString_GET_SIZE(cres)); 1287 Py_DECREF(cres); 1288 cres = ctr; 1289 self->pendingsize = 0; 1290 } 1291 1292 rsize = PyString_GET_SIZE(cres); 1293 if (decoder_prepare_buffer(&buf, PyString_AS_STRING(cres), 1294 rsize) != 0) 1295 goto errorexit; 1296 1297 if (rsize > 0 && decoder_feed_buffer( 1298 (MultibyteStatefulDecoderContext *)self, &buf)) 1299 goto errorexit; 1300 1301 if (endoffile || sizehint < 0) { 1302 if (buf.inbuf < buf.inbuf_end && 1303 multibytecodec_decerror(self->codec, &self->state, 1304 &buf, self->errors, MBERR_TOOFEW)) 1305 goto errorexit; 1306 } 1307 1308 if (buf.inbuf < buf.inbuf_end) { /* pending sequence exists */ 1309 if (decoder_append_pending(STATEFUL_DCTX(self), 1310 &buf) != 0) 1311 goto errorexit; 1312 } 1313 1314 finalsize = (Py_ssize_t)(buf.outbuf - 1315 PyUnicode_AS_UNICODE(buf.outobj)); 1316 Py_DECREF(cres); 1317 cres = NULL; 1318 1319 if (sizehint < 0 || finalsize != 0 || rsize == 0) 1320 break; 1321 1322 sizehint = 1; /* read 1 more byte and retry */ 1323 } 1324 1325 if (finalsize != PyUnicode_GET_SIZE(buf.outobj)) 1326 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1) 1327 goto errorexit; 1328 1329 Py_XDECREF(cres); 1330 Py_XDECREF(buf.excobj); 1331 return buf.outobj; 1332 1333 errorexit: 1334 Py_XDECREF(cres); 1335 Py_XDECREF(buf.excobj); 1336 Py_XDECREF(buf.outobj); 1337 return NULL; 1338 } 1339 1340 static PyObject * 1341 mbstreamreader_read(MultibyteStreamReaderObject *self, PyObject *args) 1342 { 1343 PyObject *sizeobj = NULL; 1344 Py_ssize_t size; 1345 1346 if (!PyArg_UnpackTuple(args, "read", 0, 1, &sizeobj)) 1347 return NULL; 1348 1349 if (sizeobj == Py_None || sizeobj == NULL) 1350 size = -1; 1351 else if (PyInt_Check(sizeobj)) 1352 size = PyInt_AsSsize_t(sizeobj); 1353 else { 1354 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer"); 1355 return NULL; 1356 } 1357 1358 return mbstreamreader_iread(self, "read", size); 1359 } 1360 1361 static PyObject * 1362 mbstreamreader_readline(MultibyteStreamReaderObject *self, PyObject *args) 1363 { 1364 PyObject *sizeobj = NULL; 1365 Py_ssize_t size; 1366 1367 if (!PyArg_UnpackTuple(args, "readline", 0, 1, &sizeobj)) 1368 return NULL; 1369 1370 if (sizeobj == Py_None || sizeobj == NULL) 1371 size = -1; 1372 else if (PyInt_Check(sizeobj)) 1373 size = PyInt_AsSsize_t(sizeobj); 1374 else { 1375 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer"); 1376 return NULL; 1377 } 1378 1379 return mbstreamreader_iread(self, "readline", size); 1380 } 1381 1382 static PyObject * 1383 mbstreamreader_readlines(MultibyteStreamReaderObject *self, PyObject *args) 1384 { 1385 PyObject *sizehintobj = NULL, *r, *sr; 1386 Py_ssize_t sizehint; 1387 1388 if (!PyArg_UnpackTuple(args, "readlines", 0, 1, &sizehintobj)) 1389 return NULL; 1390 1391 if (sizehintobj == Py_None || sizehintobj == NULL) 1392 sizehint = -1; 1393 else if (PyInt_Check(sizehintobj)) 1394 sizehint = PyInt_AsSsize_t(sizehintobj); 1395 else { 1396 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer"); 1397 return NULL; 1398 } 1399 1400 r = mbstreamreader_iread(self, "read", sizehint); 1401 if (r == NULL) 1402 return NULL; 1403 1404 sr = PyUnicode_Splitlines(r, 1); 1405 Py_DECREF(r); 1406 return sr; 1407 } 1408 1409 static PyObject * 1410 mbstreamreader_reset(MultibyteStreamReaderObject *self) 1411 { 1412 if (self->codec->decreset != NULL && 1413 self->codec->decreset(&self->state, self->codec->config) != 0) 1414 return NULL; 1415 self->pendingsize = 0; 1416 1417 Py_RETURN_NONE; 1418 } 1419 1420 static struct PyMethodDef mbstreamreader_methods[] = { 1421 {"read", (PyCFunction)mbstreamreader_read, 1422 METH_VARARGS, NULL}, 1423 {"readline", (PyCFunction)mbstreamreader_readline, 1424 METH_VARARGS, NULL}, 1425 {"readlines", (PyCFunction)mbstreamreader_readlines, 1426 METH_VARARGS, NULL}, 1427 {"reset", (PyCFunction)mbstreamreader_reset, 1428 METH_NOARGS, NULL}, 1429 {NULL, NULL}, 1430 }; 1431 1432 static PyMemberDef mbstreamreader_members[] = { 1433 {"stream", T_OBJECT, 1434 offsetof(MultibyteStreamReaderObject, stream), 1435 READONLY, NULL}, 1436 {NULL,} 1437 }; 1438 1439 static PyObject * 1440 mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds) 1441 { 1442 MultibyteStreamReaderObject *self; 1443 PyObject *stream, *codec = NULL; 1444 char *errors = NULL; 1445 1446 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamReader", 1447 streamkwarglist, &stream, &errors)) 1448 return NULL; 1449 1450 self = (MultibyteStreamReaderObject *)type->tp_alloc(type, 0); 1451 if (self == NULL) 1452 return NULL; 1453 1454 codec = PyObject_GetAttrString((PyObject *)type, "codec"); 1455 if (codec == NULL) 1456 goto errorexit; 1457 if (!MultibyteCodec_Check(codec)) { 1458 PyErr_SetString(PyExc_TypeError, "codec is unexpected type"); 1459 goto errorexit; 1460 } 1461 1462 self->codec = ((MultibyteCodecObject *)codec)->codec; 1463 self->stream = stream; 1464 Py_INCREF(stream); 1465 self->pendingsize = 0; 1466 self->errors = internal_error_callback(errors); 1467 if (self->errors == NULL) 1468 goto errorexit; 1469 if (self->codec->decinit != NULL && 1470 self->codec->decinit(&self->state, self->codec->config) != 0) 1471 goto errorexit; 1472 1473 Py_DECREF(codec); 1474 return (PyObject *)self; 1475 1476 errorexit: 1477 Py_XDECREF(self); 1478 Py_XDECREF(codec); 1479 return NULL; 1480 } 1481 1482 static int 1483 mbstreamreader_init(PyObject *self, PyObject *args, PyObject *kwds) 1484 { 1485 return 0; 1486 } 1487 1488 static int 1489 mbstreamreader_traverse(MultibyteStreamReaderObject *self, 1490 visitproc visit, void *arg) 1491 { 1492 if (ERROR_ISCUSTOM(self->errors)) 1493 Py_VISIT(self->errors); 1494 Py_VISIT(self->stream); 1495 return 0; 1496 } 1497 1498 static void 1499 mbstreamreader_dealloc(MultibyteStreamReaderObject *self) 1500 { 1501 PyObject_GC_UnTrack(self); 1502 ERROR_DECREF(self->errors); 1503 Py_XDECREF(self->stream); 1504 Py_TYPE(self)->tp_free(self); 1505 } 1506 1507 static PyTypeObject MultibyteStreamReader_Type = { 1508 PyVarObject_HEAD_INIT(NULL, 0) 1509 "MultibyteStreamReader", /* tp_name */ 1510 sizeof(MultibyteStreamReaderObject), /* tp_basicsize */ 1511 0, /* tp_itemsize */ 1512 /* methods */ 1513 (destructor)mbstreamreader_dealloc, /* tp_dealloc */ 1514 0, /* tp_print */ 1515 0, /* tp_getattr */ 1516 0, /* tp_setattr */ 1517 0, /* tp_compare */ 1518 0, /* tp_repr */ 1519 0, /* tp_as_number */ 1520 0, /* tp_as_sequence */ 1521 0, /* tp_as_mapping */ 1522 0, /* tp_hash */ 1523 0, /* tp_call */ 1524 0, /* tp_str */ 1525 PyObject_GenericGetAttr, /* tp_getattro */ 1526 0, /* tp_setattro */ 1527 0, /* tp_as_buffer */ 1528 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC 1529 | Py_TPFLAGS_BASETYPE, /* tp_flags */ 1530 0, /* tp_doc */ 1531 (traverseproc)mbstreamreader_traverse, /* tp_traverse */ 1532 0, /* tp_clear */ 1533 0, /* tp_richcompare */ 1534 0, /* tp_weaklistoffset */ 1535 0, /* tp_iter */ 1536 0, /* tp_iterext */ 1537 mbstreamreader_methods, /* tp_methods */ 1538 mbstreamreader_members, /* tp_members */ 1539 codecctx_getsets, /* tp_getset */ 1540 0, /* tp_base */ 1541 0, /* tp_dict */ 1542 0, /* tp_descr_get */ 1543 0, /* tp_descr_set */ 1544 0, /* tp_dictoffset */ 1545 mbstreamreader_init, /* tp_init */ 1546 0, /* tp_alloc */ 1547 mbstreamreader_new, /* tp_new */ 1548 }; 1549 1550 1551 /** 1552 * MultibyteStreamWriter object 1553 */ 1554 1555 static int 1556 mbstreamwriter_iwrite(MultibyteStreamWriterObject *self, 1557 PyObject *unistr) 1558 { 1559 PyObject *str, *wr; 1560 1561 str = encoder_encode_stateful(STATEFUL_ECTX(self), unistr, 0); 1562 if (str == NULL) 1563 return -1; 1564 1565 wr = PyObject_CallMethod(self->stream, "write", "O", str); 1566 Py_DECREF(str); 1567 if (wr == NULL) 1568 return -1; 1569 1570 Py_DECREF(wr); 1571 return 0; 1572 } 1573 1574 static PyObject * 1575 mbstreamwriter_write(MultibyteStreamWriterObject *self, PyObject *strobj) 1576 { 1577 if (mbstreamwriter_iwrite(self, strobj)) 1578 return NULL; 1579 else 1580 Py_RETURN_NONE; 1581 } 1582 1583 static PyObject * 1584 mbstreamwriter_writelines(MultibyteStreamWriterObject *self, PyObject *lines) 1585 { 1586 PyObject *strobj; 1587 int i, r; 1588 1589 if (!PySequence_Check(lines)) { 1590 PyErr_SetString(PyExc_TypeError, 1591 "arg must be a sequence object"); 1592 return NULL; 1593 } 1594 1595 for (i = 0; i < PySequence_Length(lines); i++) { 1596 /* length can be changed even within this loop */ 1597 strobj = PySequence_GetItem(lines, i); 1598 if (strobj == NULL) 1599 return NULL; 1600 1601 r = mbstreamwriter_iwrite(self, strobj); 1602 Py_DECREF(strobj); 1603 if (r == -1) 1604 return NULL; 1605 } 1606 1607 Py_RETURN_NONE; 1608 } 1609 1610 static PyObject * 1611 mbstreamwriter_reset(MultibyteStreamWriterObject *self) 1612 { 1613 const Py_UNICODE *pending; 1614 PyObject *pwrt; 1615 1616 pending = self->pending; 1617 pwrt = multibytecodec_encode(self->codec, &self->state, 1618 &pending, self->pendingsize, self->errors, 1619 MBENC_FLUSH | MBENC_RESET); 1620 /* some pending buffer can be truncated when UnicodeEncodeError is 1621 * raised on 'strict' mode. but, 'reset' method is designed to 1622 * reset the pending buffer or states so failed string sequence 1623 * ought to be missed */ 1624 self->pendingsize = 0; 1625 if (pwrt == NULL) 1626 return NULL; 1627 1628 if (PyString_Size(pwrt) > 0) { 1629 PyObject *wr; 1630 wr = PyObject_CallMethod(self->stream, "write", "O", pwrt); 1631 if (wr == NULL) { 1632 Py_DECREF(pwrt); 1633 return NULL; 1634 } 1635 } 1636 Py_DECREF(pwrt); 1637 1638 Py_RETURN_NONE; 1639 } 1640 1641 static PyObject * 1642 mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) 1643 { 1644 MultibyteStreamWriterObject *self; 1645 PyObject *stream, *codec = NULL; 1646 char *errors = NULL; 1647 1648 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamWriter", 1649 streamkwarglist, &stream, &errors)) 1650 return NULL; 1651 1652 self = (MultibyteStreamWriterObject *)type->tp_alloc(type, 0); 1653 if (self == NULL) 1654 return NULL; 1655 1656 codec = PyObject_GetAttrString((PyObject *)type, "codec"); 1657 if (codec == NULL) 1658 goto errorexit; 1659 if (!MultibyteCodec_Check(codec)) { 1660 PyErr_SetString(PyExc_TypeError, "codec is unexpected type"); 1661 goto errorexit; 1662 } 1663 1664 self->codec = ((MultibyteCodecObject *)codec)->codec; 1665 self->stream = stream; 1666 Py_INCREF(stream); 1667 self->pendingsize = 0; 1668 self->errors = internal_error_callback(errors); 1669 if (self->errors == NULL) 1670 goto errorexit; 1671 if (self->codec->encinit != NULL && 1672 self->codec->encinit(&self->state, self->codec->config) != 0) 1673 goto errorexit; 1674 1675 Py_DECREF(codec); 1676 return (PyObject *)self; 1677 1678 errorexit: 1679 Py_XDECREF(self); 1680 Py_XDECREF(codec); 1681 return NULL; 1682 } 1683 1684 static int 1685 mbstreamwriter_init(PyObject *self, PyObject *args, PyObject *kwds) 1686 { 1687 return 0; 1688 } 1689 1690 static int 1691 mbstreamwriter_traverse(MultibyteStreamWriterObject *self, 1692 visitproc visit, void *arg) 1693 { 1694 if (ERROR_ISCUSTOM(self->errors)) 1695 Py_VISIT(self->errors); 1696 Py_VISIT(self->stream); 1697 return 0; 1698 } 1699 1700 static void 1701 mbstreamwriter_dealloc(MultibyteStreamWriterObject *self) 1702 { 1703 PyObject_GC_UnTrack(self); 1704 ERROR_DECREF(self->errors); 1705 Py_XDECREF(self->stream); 1706 Py_TYPE(self)->tp_free(self); 1707 } 1708 1709 static struct PyMethodDef mbstreamwriter_methods[] = { 1710 {"write", (PyCFunction)mbstreamwriter_write, 1711 METH_O, NULL}, 1712 {"writelines", (PyCFunction)mbstreamwriter_writelines, 1713 METH_O, NULL}, 1714 {"reset", (PyCFunction)mbstreamwriter_reset, 1715 METH_NOARGS, NULL}, 1716 {NULL, NULL}, 1717 }; 1718 1719 static PyMemberDef mbstreamwriter_members[] = { 1720 {"stream", T_OBJECT, 1721 offsetof(MultibyteStreamWriterObject, stream), 1722 READONLY, NULL}, 1723 {NULL,} 1724 }; 1725 1726 static PyTypeObject MultibyteStreamWriter_Type = { 1727 PyVarObject_HEAD_INIT(NULL, 0) 1728 "MultibyteStreamWriter", /* tp_name */ 1729 sizeof(MultibyteStreamWriterObject), /* tp_basicsize */ 1730 0, /* tp_itemsize */ 1731 /* methods */ 1732 (destructor)mbstreamwriter_dealloc, /* tp_dealloc */ 1733 0, /* tp_print */ 1734 0, /* tp_getattr */ 1735 0, /* tp_setattr */ 1736 0, /* tp_compare */ 1737 0, /* tp_repr */ 1738 0, /* tp_as_number */ 1739 0, /* tp_as_sequence */ 1740 0, /* tp_as_mapping */ 1741 0, /* tp_hash */ 1742 0, /* tp_call */ 1743 0, /* tp_str */ 1744 PyObject_GenericGetAttr, /* tp_getattro */ 1745 0, /* tp_setattro */ 1746 0, /* tp_as_buffer */ 1747 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC 1748 | Py_TPFLAGS_BASETYPE, /* tp_flags */ 1749 0, /* tp_doc */ 1750 (traverseproc)mbstreamwriter_traverse, /* tp_traverse */ 1751 0, /* tp_clear */ 1752 0, /* tp_richcompare */ 1753 0, /* tp_weaklistoffset */ 1754 0, /* tp_iter */ 1755 0, /* tp_iterext */ 1756 mbstreamwriter_methods, /* tp_methods */ 1757 mbstreamwriter_members, /* tp_members */ 1758 codecctx_getsets, /* tp_getset */ 1759 0, /* tp_base */ 1760 0, /* tp_dict */ 1761 0, /* tp_descr_get */ 1762 0, /* tp_descr_set */ 1763 0, /* tp_dictoffset */ 1764 mbstreamwriter_init, /* tp_init */ 1765 0, /* tp_alloc */ 1766 mbstreamwriter_new, /* tp_new */ 1767 }; 1768 1769 1770 /** 1771 * Exposed factory function 1772 */ 1773 1774 static PyObject * 1775 __create_codec(PyObject *ignore, PyObject *arg) 1776 { 1777 MultibyteCodecObject *self; 1778 MultibyteCodec *codec; 1779 1780 if (!PyCapsule_IsValid(arg, PyMultibyteCodec_CAPSULE_NAME)) { 1781 PyErr_SetString(PyExc_ValueError, "argument type invalid"); 1782 return NULL; 1783 } 1784 1785 codec = PyCapsule_GetPointer(arg, PyMultibyteCodec_CAPSULE_NAME); 1786 if (codec->codecinit != NULL && codec->codecinit(codec->config) != 0) 1787 return NULL; 1788 1789 self = PyObject_New(MultibyteCodecObject, &MultibyteCodec_Type); 1790 if (self == NULL) 1791 return NULL; 1792 self->codec = codec; 1793 1794 return (PyObject *)self; 1795 } 1796 1797 static struct PyMethodDef __methods[] = { 1798 {"__create_codec", (PyCFunction)__create_codec, METH_O}, 1799 {NULL, NULL}, 1800 }; 1801 1802 PyMODINIT_FUNC 1803 init_multibytecodec(void) 1804 { 1805 int i; 1806 PyObject *m; 1807 PyTypeObject *typelist[] = { 1808 &MultibyteIncrementalEncoder_Type, 1809 &MultibyteIncrementalDecoder_Type, 1810 &MultibyteStreamReader_Type, 1811 &MultibyteStreamWriter_Type, 1812 NULL 1813 }; 1814 1815 if (PyType_Ready(&MultibyteCodec_Type) < 0) 1816 return; 1817 1818 m = Py_InitModule("_multibytecodec", __methods); 1819 if (m == NULL) 1820 return; 1821 1822 for (i = 0; typelist[i] != NULL; i++) { 1823 if (PyType_Ready(typelist[i]) < 0) 1824 return; 1825 Py_INCREF(typelist[i]); 1826 PyModule_AddObject(m, typelist[i]->tp_name, 1827 (PyObject *)typelist[i]); 1828 } 1829 1830 if (PyErr_Occurred()) 1831 Py_FatalError("can't initialize the _multibytecodec module"); 1832 } 1833
