1 /*************************************************************************** 2 * _ _ ____ _ 3 * Project ___| | | | _ \| | 4 * / __| | | | |_) | | 5 * | (__| |_| | _ <| |___ 6 * \___|\___/|_| \_\_____| 7 * 8 * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel (at) haxx.se>, et al. 9 * 10 * This software is licensed as described in the file COPYING, which 11 * you should have received as part of this distribution. The terms 12 * are also available at https://curl.haxx.se/docs/copyright.html. 13 * 14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell 15 * copies of the Software, and permit persons to whom the Software is 16 * furnished to do so, under the terms of the COPYING file. 17 * 18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 19 * KIND, either express or implied. 20 * 21 ***************************************************************************/ 22 23 #include "curl_setup.h" 24 #include "strtoofft.h" 25 26 #ifdef HAVE_NETINET_IN_H 27 #include <netinet/in.h> 28 #endif 29 #ifdef HAVE_NETDB_H 30 #include <netdb.h> 31 #endif 32 #ifdef HAVE_ARPA_INET_H 33 #include <arpa/inet.h> 34 #endif 35 #ifdef HAVE_NET_IF_H 36 #include <net/if.h> 37 #endif 38 #ifdef HAVE_SYS_IOCTL_H 39 #include <sys/ioctl.h> 40 #endif 41 #ifdef HAVE_SIGNAL_H 42 #include <signal.h> 43 #endif 44 45 #ifdef HAVE_SYS_PARAM_H 46 #include <sys/param.h> 47 #endif 48 49 #ifdef HAVE_SYS_SELECT_H 50 #include <sys/select.h> 51 #endif 52 53 #ifndef HAVE_SOCKET 54 #error "We can't compile without socket() support!" 55 #endif 56 57 #include "urldata.h" 58 #include <curl/curl.h> 59 #include "netrc.h" 60 61 #include "content_encoding.h" 62 #include "hostip.h" 63 #include "transfer.h" 64 #include "sendf.h" 65 #include "speedcheck.h" 66 #include "progress.h" 67 #include "http.h" 68 #include "url.h" 69 #include "getinfo.h" 70 #include "vtls/vtls.h" 71 #include "select.h" 72 #include "multiif.h" 73 #include "connect.h" 74 #include "non-ascii.h" 75 #include "http2.h" 76 77 /* The last 3 #include files should be in this order */ 78 #include "curl_printf.h" 79 #include "curl_memory.h" 80 #include "memdebug.h" 81 82 /* 83 * This function will call the read callback to fill our buffer with data 84 * to upload. 85 */ 86 CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp) 87 { 88 struct Curl_easy *data = conn->data; 89 size_t buffersize = (size_t)bytes; 90 int nread; 91 #ifdef CURL_DOES_CONVERSIONS 92 bool sending_http_headers = FALSE; 93 94 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) { 95 const struct HTTP *http = data->req.protop; 96 97 if(http->sending == HTTPSEND_REQUEST) 98 /* We're sending the HTTP request headers, not the data. 99 Remember that so we don't re-translate them into garbage. */ 100 sending_http_headers = TRUE; 101 } 102 #endif 103 104 if(data->req.upload_chunky) { 105 /* if chunked Transfer-Encoding */ 106 buffersize -= (8 + 2 + 2); /* 32bit hex + CRLF + CRLF */ 107 data->req.upload_fromhere += (8 + 2); /* 32bit hex + CRLF */ 108 } 109 110 /* this function returns a size_t, so we typecast to int to prevent warnings 111 with picky compilers */ 112 nread = (int)data->state.fread_func(data->req.upload_fromhere, 1, 113 buffersize, data->state.in); 114 115 if(nread == CURL_READFUNC_ABORT) { 116 failf(data, "operation aborted by callback"); 117 *nreadp = 0; 118 return CURLE_ABORTED_BY_CALLBACK; 119 } 120 else if(nread == CURL_READFUNC_PAUSE) { 121 122 if(conn->handler->flags & PROTOPT_NONETWORK) { 123 /* protocols that work without network cannot be paused. This is 124 actually only FILE:// just now, and it can't pause since the transfer 125 isn't done using the "normal" procedure. */ 126 failf(data, "Read callback asked for PAUSE when not supported!"); 127 return CURLE_READ_ERROR; 128 } 129 else { 130 struct SingleRequest *k = &data->req; 131 /* CURL_READFUNC_PAUSE pauses read callbacks that feed socket writes */ 132 k->keepon |= KEEP_SEND_PAUSE; /* mark socket send as paused */ 133 if(data->req.upload_chunky) { 134 /* Back out the preallocation done above */ 135 data->req.upload_fromhere -= (8 + 2); 136 } 137 *nreadp = 0; 138 } 139 return CURLE_OK; /* nothing was read */ 140 } 141 else if((size_t)nread > buffersize) { 142 /* the read function returned a too large value */ 143 *nreadp = 0; 144 failf(data, "read function returned funny value"); 145 return CURLE_READ_ERROR; 146 } 147 148 if(!data->req.forbidchunk && data->req.upload_chunky) { 149 /* if chunked Transfer-Encoding 150 * build chunk: 151 * 152 * <HEX SIZE> CRLF 153 * <DATA> CRLF 154 */ 155 /* On non-ASCII platforms the <DATA> may or may not be 156 translated based on set.prefer_ascii while the protocol 157 portion must always be translated to the network encoding. 158 To further complicate matters, line end conversion might be 159 done later on, so we need to prevent CRLFs from becoming 160 CRCRLFs if that's the case. To do this we use bare LFs 161 here, knowing they'll become CRLFs later on. 162 */ 163 164 char hexbuffer[11]; 165 const char *endofline_native; 166 const char *endofline_network; 167 int hexlen; 168 169 if( 170 #ifdef CURL_DO_LINEEND_CONV 171 (data->set.prefer_ascii) || 172 #endif 173 (data->set.crlf)) { 174 /* \n will become \r\n later on */ 175 endofline_native = "\n"; 176 endofline_network = "\x0a"; 177 } 178 else { 179 endofline_native = "\r\n"; 180 endofline_network = "\x0d\x0a"; 181 } 182 hexlen = snprintf(hexbuffer, sizeof(hexbuffer), 183 "%x%s", nread, endofline_native); 184 185 /* move buffer pointer */ 186 data->req.upload_fromhere -= hexlen; 187 nread += hexlen; 188 189 /* copy the prefix to the buffer, leaving out the NUL */ 190 memcpy(data->req.upload_fromhere, hexbuffer, hexlen); 191 192 /* always append ASCII CRLF to the data */ 193 memcpy(data->req.upload_fromhere + nread, 194 endofline_network, 195 strlen(endofline_network)); 196 197 #ifdef CURL_DOES_CONVERSIONS 198 CURLcode result; 199 int length; 200 if(data->set.prefer_ascii) { 201 /* translate the protocol and data */ 202 length = nread; 203 } 204 else { 205 /* just translate the protocol portion */ 206 length = strlen(hexbuffer); 207 } 208 result = Curl_convert_to_network(data, data->req.upload_fromhere, length); 209 /* Curl_convert_to_network calls failf if unsuccessful */ 210 if(result) 211 return result; 212 #endif /* CURL_DOES_CONVERSIONS */ 213 214 if((nread - hexlen) == 0) 215 /* mark this as done once this chunk is transferred */ 216 data->req.upload_done = TRUE; 217 218 nread+=(int)strlen(endofline_native); /* for the added end of line */ 219 } 220 #ifdef CURL_DOES_CONVERSIONS 221 else if((data->set.prefer_ascii) && (!sending_http_headers)) { 222 CURLcode result; 223 result = Curl_convert_to_network(data, data->req.upload_fromhere, nread); 224 /* Curl_convert_to_network calls failf if unsuccessful */ 225 if(result) 226 return result; 227 } 228 #endif /* CURL_DOES_CONVERSIONS */ 229 230 *nreadp = nread; 231 232 return CURLE_OK; 233 } 234 235 236 /* 237 * Curl_readrewind() rewinds the read stream. This is typically used for HTTP 238 * POST/PUT with multi-pass authentication when a sending was denied and a 239 * resend is necessary. 240 */ 241 CURLcode Curl_readrewind(struct connectdata *conn) 242 { 243 struct Curl_easy *data = conn->data; 244 245 conn->bits.rewindaftersend = FALSE; /* we rewind now */ 246 247 /* explicitly switch off sending data on this connection now since we are 248 about to restart a new transfer and thus we want to avoid inadvertently 249 sending more data on the existing connection until the next transfer 250 starts */ 251 data->req.keepon &= ~KEEP_SEND; 252 253 /* We have sent away data. If not using CURLOPT_POSTFIELDS or 254 CURLOPT_HTTPPOST, call app to rewind 255 */ 256 if(data->set.postfields || 257 (data->set.httpreq == HTTPREQ_POST_FORM)) 258 ; /* do nothing */ 259 else { 260 if(data->set.seek_func) { 261 int err; 262 263 err = (data->set.seek_func)(data->set.seek_client, 0, SEEK_SET); 264 if(err) { 265 failf(data, "seek callback returned error %d", (int)err); 266 return CURLE_SEND_FAIL_REWIND; 267 } 268 } 269 else if(data->set.ioctl_func) { 270 curlioerr err; 271 272 err = (data->set.ioctl_func)(data, CURLIOCMD_RESTARTREAD, 273 data->set.ioctl_client); 274 infof(data, "the ioctl callback returned %d\n", (int)err); 275 276 if(err) { 277 /* FIXME: convert to a human readable error message */ 278 failf(data, "ioctl callback returned error %d", (int)err); 279 return CURLE_SEND_FAIL_REWIND; 280 } 281 } 282 else { 283 /* If no CURLOPT_READFUNCTION is used, we know that we operate on a 284 given FILE * stream and we can actually attempt to rewind that 285 ourselves with fseek() */ 286 if(data->state.fread_func == (curl_read_callback)fread) { 287 if(-1 != fseek(data->state.in, 0, SEEK_SET)) 288 /* successful rewind */ 289 return CURLE_OK; 290 } 291 292 /* no callback set or failure above, makes us fail at once */ 293 failf(data, "necessary data rewind wasn't possible"); 294 return CURLE_SEND_FAIL_REWIND; 295 } 296 } 297 return CURLE_OK; 298 } 299 300 static int data_pending(const struct connectdata *conn) 301 { 302 /* in the case of libssh2, we can never be really sure that we have emptied 303 its internal buffers so we MUST always try until we get EAGAIN back */ 304 return conn->handler->protocol&(CURLPROTO_SCP|CURLPROTO_SFTP) || 305 #if defined(USE_NGHTTP2) 306 Curl_ssl_data_pending(conn, FIRSTSOCKET) || 307 /* For HTTP/2, we may read up everything including responde body 308 with header fields in Curl_http_readwrite_headers. If no 309 content-length is provided, curl waits for the connection 310 close, which we emulate it using conn->proto.httpc.closed = 311 TRUE. The thing is if we read everything, then http2_recv won't 312 be called and we cannot signal the HTTP/2 stream has closed. As 313 a workaround, we return nonzero here to call http2_recv. */ 314 ((conn->handler->protocol&PROTO_FAMILY_HTTP) && conn->httpversion == 20); 315 #else 316 Curl_ssl_data_pending(conn, FIRSTSOCKET); 317 #endif 318 } 319 320 static void read_rewind(struct connectdata *conn, 321 size_t thismuch) 322 { 323 DEBUGASSERT(conn->read_pos >= thismuch); 324 325 conn->read_pos -= thismuch; 326 conn->bits.stream_was_rewound = TRUE; 327 328 #ifdef DEBUGBUILD 329 { 330 char buf[512 + 1]; 331 size_t show; 332 333 show = CURLMIN(conn->buf_len - conn->read_pos, sizeof(buf)-1); 334 if(conn->master_buffer) { 335 memcpy(buf, conn->master_buffer + conn->read_pos, show); 336 buf[show] = '\0'; 337 } 338 else { 339 buf[0] = '\0'; 340 } 341 342 DEBUGF(infof(conn->data, 343 "Buffer after stream rewind (read_pos = %zu): [%s]\n", 344 conn->read_pos, buf)); 345 } 346 #endif 347 } 348 349 /* 350 * Check to see if CURLOPT_TIMECONDITION was met by comparing the time of the 351 * remote document with the time provided by CURLOPT_TIMEVAL 352 */ 353 bool Curl_meets_timecondition(struct Curl_easy *data, time_t timeofdoc) 354 { 355 if((timeofdoc == 0) || (data->set.timevalue == 0)) 356 return TRUE; 357 358 switch(data->set.timecondition) { 359 case CURL_TIMECOND_IFMODSINCE: 360 default: 361 if(timeofdoc <= data->set.timevalue) { 362 infof(data, 363 "The requested document is not new enough\n"); 364 data->info.timecond = TRUE; 365 return FALSE; 366 } 367 break; 368 case CURL_TIMECOND_IFUNMODSINCE: 369 if(timeofdoc >= data->set.timevalue) { 370 infof(data, 371 "The requested document is not old enough\n"); 372 data->info.timecond = TRUE; 373 return FALSE; 374 } 375 break; 376 } 377 378 return TRUE; 379 } 380 381 /* 382 * Go ahead and do a read if we have a readable socket or if 383 * the stream was rewound (in which case we have data in a 384 * buffer) 385 * 386 * return '*comeback' TRUE if we didn't properly drain the socket so this 387 * function should get called again without select() or similar in between! 388 */ 389 static CURLcode readwrite_data(struct Curl_easy *data, 390 struct connectdata *conn, 391 struct SingleRequest *k, 392 int *didwhat, bool *done, 393 bool *comeback) 394 { 395 CURLcode result = CURLE_OK; 396 ssize_t nread; /* number of bytes read */ 397 size_t excess = 0; /* excess bytes read */ 398 bool is_empty_data = FALSE; 399 bool readmore = FALSE; /* used by RTP to signal for more data */ 400 int maxloops = 100; 401 402 *done = FALSE; 403 *comeback = FALSE; 404 405 /* This is where we loop until we have read everything there is to 406 read or we get a CURLE_AGAIN */ 407 do { 408 size_t buffersize = data->set.buffer_size? 409 data->set.buffer_size : BUFSIZE; 410 size_t bytestoread = buffersize; 411 412 if( 413 #if defined(USE_NGHTTP2) 414 /* For HTTP/2, read data without caring about the content 415 length. This is safe because body in HTTP/2 is always 416 segmented thanks to its framing layer. Meanwhile, we have to 417 call Curl_read to ensure that http2_handle_stream_close is 418 called when we read all incoming bytes for a particular 419 stream. */ 420 !((conn->handler->protocol & PROTO_FAMILY_HTTP) && 421 conn->httpversion == 20) && 422 #endif 423 k->size != -1 && !k->header) { 424 /* make sure we don't read "too much" if we can help it since we 425 might be pipelining and then someone else might want to read what 426 follows! */ 427 curl_off_t totalleft = k->size - k->bytecount; 428 if(totalleft < (curl_off_t)bytestoread) 429 bytestoread = (size_t)totalleft; 430 } 431 432 if(bytestoread) { 433 /* receive data from the network! */ 434 result = Curl_read(conn, conn->sockfd, k->buf, bytestoread, &nread); 435 436 /* read would've blocked */ 437 if(CURLE_AGAIN == result) 438 break; /* get out of loop */ 439 440 if(result>0) 441 return result; 442 } 443 else { 444 /* read nothing but since we wanted nothing we consider this an OK 445 situation to proceed from */ 446 DEBUGF(infof(data, "readwrite_data: we're done!\n")); 447 nread = 0; 448 } 449 450 if((k->bytecount == 0) && (k->writebytecount == 0)) { 451 Curl_pgrsTime(data, TIMER_STARTTRANSFER); 452 if(k->exp100 > EXP100_SEND_DATA) 453 /* set time stamp to compare with when waiting for the 100 */ 454 k->start100 = Curl_tvnow(); 455 } 456 457 *didwhat |= KEEP_RECV; 458 /* indicates data of zero size, i.e. empty file */ 459 is_empty_data = ((nread == 0) && (k->bodywrites == 0)) ? TRUE : FALSE; 460 461 /* NUL terminate, allowing string ops to be used */ 462 if(0 < nread || is_empty_data) { 463 k->buf[nread] = 0; 464 } 465 else if(0 >= nread) { 466 /* if we receive 0 or less here, the server closed the connection 467 and we bail out from this! */ 468 DEBUGF(infof(data, "nread <= 0, server closed connection, bailing\n")); 469 k->keepon &= ~KEEP_RECV; 470 break; 471 } 472 473 /* Default buffer to use when we write the buffer, it may be changed 474 in the flow below before the actual storing is done. */ 475 k->str = k->buf; 476 477 if(conn->handler->readwrite) { 478 result = conn->handler->readwrite(data, conn, &nread, &readmore); 479 if(result) 480 return result; 481 if(readmore) 482 break; 483 } 484 485 #ifndef CURL_DISABLE_HTTP 486 /* Since this is a two-state thing, we check if we are parsing 487 headers at the moment or not. */ 488 if(k->header) { 489 /* we are in parse-the-header-mode */ 490 bool stop_reading = FALSE; 491 result = Curl_http_readwrite_headers(data, conn, &nread, &stop_reading); 492 if(result) 493 return result; 494 495 if(conn->handler->readwrite && 496 (k->maxdownload <= 0 && nread > 0)) { 497 result = conn->handler->readwrite(data, conn, &nread, &readmore); 498 if(result) 499 return result; 500 if(readmore) 501 break; 502 } 503 504 if(stop_reading) { 505 /* We've stopped dealing with input, get out of the do-while loop */ 506 507 if(nread > 0) { 508 if(Curl_pipeline_wanted(conn->data->multi, CURLPIPE_HTTP1)) { 509 infof(data, 510 "Rewinding stream by : %zd" 511 " bytes on url %s (zero-length body)\n", 512 nread, data->state.path); 513 read_rewind(conn, (size_t)nread); 514 } 515 else { 516 infof(data, 517 "Excess found in a non pipelined read:" 518 " excess = %zd" 519 " url = %s (zero-length body)\n", 520 nread, data->state.path); 521 } 522 } 523 524 break; 525 } 526 } 527 #endif /* CURL_DISABLE_HTTP */ 528 529 530 /* This is not an 'else if' since it may be a rest from the header 531 parsing, where the beginning of the buffer is headers and the end 532 is non-headers. */ 533 if(k->str && !k->header && (nread > 0 || is_empty_data)) { 534 535 if(data->set.opt_no_body) { 536 /* data arrives although we want none, bail out */ 537 streamclose(conn, "ignoring body"); 538 *done = TRUE; 539 return CURLE_WEIRD_SERVER_REPLY; 540 } 541 542 #ifndef CURL_DISABLE_HTTP 543 if(0 == k->bodywrites && !is_empty_data) { 544 /* These checks are only made the first time we are about to 545 write a piece of the body */ 546 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) { 547 /* HTTP-only checks */ 548 549 if(data->req.newurl) { 550 if(conn->bits.close) { 551 /* Abort after the headers if "follow Location" is set 552 and we're set to close anyway. */ 553 k->keepon &= ~KEEP_RECV; 554 *done = TRUE; 555 return CURLE_OK; 556 } 557 /* We have a new url to load, but since we want to be able 558 to re-use this connection properly, we read the full 559 response in "ignore more" */ 560 k->ignorebody = TRUE; 561 infof(data, "Ignoring the response-body\n"); 562 } 563 if(data->state.resume_from && !k->content_range && 564 (data->set.httpreq==HTTPREQ_GET) && 565 !k->ignorebody) { 566 567 if(k->size == data->state.resume_from) { 568 /* The resume point is at the end of file, consider this fine 569 even if it doesn't allow resume from here. */ 570 infof(data, "The entire document is already downloaded"); 571 connclose(conn, "already downloaded"); 572 /* Abort download */ 573 k->keepon &= ~KEEP_RECV; 574 *done = TRUE; 575 return CURLE_OK; 576 } 577 578 /* we wanted to resume a download, although the server doesn't 579 * seem to support this and we did this with a GET (if it 580 * wasn't a GET we did a POST or PUT resume) */ 581 failf(data, "HTTP server doesn't seem to support " 582 "byte ranges. Cannot resume."); 583 return CURLE_RANGE_ERROR; 584 } 585 586 if(data->set.timecondition && !data->state.range) { 587 /* A time condition has been set AND no ranges have been 588 requested. This seems to be what chapter 13.3.4 of 589 RFC 2616 defines to be the correct action for a 590 HTTP/1.1 client */ 591 592 if(!Curl_meets_timecondition(data, k->timeofdoc)) { 593 *done = TRUE; 594 /* We're simulating a http 304 from server so we return 595 what should have been returned from the server */ 596 data->info.httpcode = 304; 597 infof(data, "Simulate a HTTP 304 response!\n"); 598 /* we abort the transfer before it is completed == we ruin the 599 re-use ability. Close the connection */ 600 connclose(conn, "Simulated 304 handling"); 601 return CURLE_OK; 602 } 603 } /* we have a time condition */ 604 605 } /* this is HTTP or RTSP */ 606 } /* this is the first time we write a body part */ 607 #endif /* CURL_DISABLE_HTTP */ 608 609 k->bodywrites++; 610 611 /* pass data to the debug function before it gets "dechunked" */ 612 if(data->set.verbose) { 613 if(k->badheader) { 614 Curl_debug(data, CURLINFO_DATA_IN, data->state.headerbuff, 615 (size_t)k->hbuflen, conn); 616 if(k->badheader == HEADER_PARTHEADER) 617 Curl_debug(data, CURLINFO_DATA_IN, 618 k->str, (size_t)nread, conn); 619 } 620 else 621 Curl_debug(data, CURLINFO_DATA_IN, 622 k->str, (size_t)nread, conn); 623 } 624 625 #ifndef CURL_DISABLE_HTTP 626 if(k->chunk) { 627 /* 628 * Here comes a chunked transfer flying and we need to decode this 629 * properly. While the name says read, this function both reads 630 * and writes away the data. The returned 'nread' holds the number 631 * of actual data it wrote to the client. 632 */ 633 634 CHUNKcode res = 635 Curl_httpchunk_read(conn, k->str, nread, &nread); 636 637 if(CHUNKE_OK < res) { 638 if(CHUNKE_WRITE_ERROR == res) { 639 failf(data, "Failed writing data"); 640 return CURLE_WRITE_ERROR; 641 } 642 failf(data, "%s in chunked-encoding", Curl_chunked_strerror(res)); 643 return CURLE_RECV_ERROR; 644 } 645 else if(CHUNKE_STOP == res) { 646 size_t dataleft; 647 /* we're done reading chunks! */ 648 k->keepon &= ~KEEP_RECV; /* read no more */ 649 650 /* There are now possibly N number of bytes at the end of the 651 str buffer that weren't written to the client. 652 653 We DO care about this data if we are pipelining. 654 Push it back to be read on the next pass. */ 655 656 dataleft = conn->chunk.dataleft; 657 if(dataleft != 0) { 658 infof(conn->data, "Leftovers after chunking: %zu bytes\n", 659 dataleft); 660 if(Curl_pipeline_wanted(conn->data->multi, CURLPIPE_HTTP1)) { 661 /* only attempt the rewind if we truly are pipelining */ 662 infof(conn->data, "Rewinding %zu bytes\n",dataleft); 663 read_rewind(conn, dataleft); 664 } 665 } 666 } 667 /* If it returned OK, we just keep going */ 668 } 669 #endif /* CURL_DISABLE_HTTP */ 670 671 /* Account for body content stored in the header buffer */ 672 if(k->badheader && !k->ignorebody) { 673 DEBUGF(infof(data, "Increasing bytecount by %zu from hbuflen\n", 674 k->hbuflen)); 675 k->bytecount += k->hbuflen; 676 } 677 678 if((-1 != k->maxdownload) && 679 (k->bytecount + nread >= k->maxdownload)) { 680 681 excess = (size_t)(k->bytecount + nread - k->maxdownload); 682 if(excess > 0 && !k->ignorebody) { 683 if(Curl_pipeline_wanted(conn->data->multi, CURLPIPE_HTTP1)) { 684 /* The 'excess' amount below can't be more than BUFSIZE which 685 always will fit in a size_t */ 686 infof(data, 687 "Rewinding stream by : %zu" 688 " bytes on url %s (size = %" CURL_FORMAT_CURL_OFF_T 689 ", maxdownload = %" CURL_FORMAT_CURL_OFF_T 690 ", bytecount = %" CURL_FORMAT_CURL_OFF_T ", nread = %zd)\n", 691 excess, data->state.path, 692 k->size, k->maxdownload, k->bytecount, nread); 693 read_rewind(conn, excess); 694 } 695 else { 696 infof(data, 697 "Excess found in a non pipelined read:" 698 " excess = %zu" 699 ", size = %" CURL_FORMAT_CURL_OFF_T 700 ", maxdownload = %" CURL_FORMAT_CURL_OFF_T 701 ", bytecount = %" CURL_FORMAT_CURL_OFF_T "\n", 702 excess, k->size, k->maxdownload, k->bytecount); 703 } 704 } 705 706 nread = (ssize_t) (k->maxdownload - k->bytecount); 707 if(nread < 0) /* this should be unusual */ 708 nread = 0; 709 710 k->keepon &= ~KEEP_RECV; /* we're done reading */ 711 } 712 713 k->bytecount += nread; 714 715 Curl_pgrsSetDownloadCounter(data, k->bytecount); 716 717 if(!k->chunk && (nread || k->badheader || is_empty_data)) { 718 /* If this is chunky transfer, it was already written */ 719 720 if(k->badheader && !k->ignorebody) { 721 /* we parsed a piece of data wrongly assuming it was a header 722 and now we output it as body instead */ 723 724 /* Don't let excess data pollute body writes */ 725 if(k->maxdownload == -1 || (curl_off_t)k->hbuflen <= k->maxdownload) 726 result = Curl_client_write(conn, CLIENTWRITE_BODY, 727 data->state.headerbuff, 728 k->hbuflen); 729 else 730 result = Curl_client_write(conn, CLIENTWRITE_BODY, 731 data->state.headerbuff, 732 (size_t)k->maxdownload); 733 734 if(result) 735 return result; 736 } 737 if(k->badheader < HEADER_ALLBAD) { 738 /* This switch handles various content encodings. If there's an 739 error here, be sure to check over the almost identical code 740 in http_chunks.c. 741 Make sure that ALL_CONTENT_ENCODINGS contains all the 742 encodings handled here. */ 743 #ifdef HAVE_LIBZ 744 switch (conn->data->set.http_ce_skip ? 745 IDENTITY : k->auto_decoding) { 746 case IDENTITY: 747 #endif 748 /* This is the default when the server sends no 749 Content-Encoding header. See Curl_readwrite_init; the 750 memset() call initializes k->auto_decoding to zero. */ 751 if(!k->ignorebody) { 752 753 #ifndef CURL_DISABLE_POP3 754 if(conn->handler->protocol&PROTO_FAMILY_POP3) 755 result = Curl_pop3_write(conn, k->str, nread); 756 else 757 #endif /* CURL_DISABLE_POP3 */ 758 759 result = Curl_client_write(conn, CLIENTWRITE_BODY, k->str, 760 nread); 761 } 762 #ifdef HAVE_LIBZ 763 break; 764 765 case DEFLATE: 766 /* Assume CLIENTWRITE_BODY; headers are not encoded. */ 767 if(!k->ignorebody) 768 result = Curl_unencode_deflate_write(conn, k, nread); 769 break; 770 771 case GZIP: 772 /* Assume CLIENTWRITE_BODY; headers are not encoded. */ 773 if(!k->ignorebody) 774 result = Curl_unencode_gzip_write(conn, k, nread); 775 break; 776 777 default: 778 failf (data, "Unrecognized content encoding type. " 779 "libcurl understands `identity', `deflate' and `gzip' " 780 "content encodings."); 781 result = CURLE_BAD_CONTENT_ENCODING; 782 break; 783 } 784 #endif 785 } 786 k->badheader = HEADER_NORMAL; /* taken care of now */ 787 788 if(result) 789 return result; 790 } 791 792 } /* if(!header and data to read) */ 793 794 if(conn->handler->readwrite && 795 (excess > 0 && !conn->bits.stream_was_rewound)) { 796 /* Parse the excess data */ 797 k->str += nread; 798 nread = (ssize_t)excess; 799 800 result = conn->handler->readwrite(data, conn, &nread, &readmore); 801 if(result) 802 return result; 803 804 if(readmore) 805 k->keepon |= KEEP_RECV; /* we're not done reading */ 806 break; 807 } 808 809 if(is_empty_data) { 810 /* if we received nothing, the server closed the connection and we 811 are done */ 812 k->keepon &= ~KEEP_RECV; 813 } 814 815 } while(data_pending(conn) && maxloops--); 816 817 if(maxloops <= 0) { 818 /* we mark it as read-again-please */ 819 conn->cselect_bits = CURL_CSELECT_IN; 820 *comeback = TRUE; 821 } 822 823 if(((k->keepon & (KEEP_RECV|KEEP_SEND)) == KEEP_SEND) && 824 conn->bits.close) { 825 /* When we've read the entire thing and the close bit is set, the server 826 may now close the connection. If there's now any kind of sending going 827 on from our side, we need to stop that immediately. */ 828 infof(data, "we are done reading and this is set to close, stop send\n"); 829 k->keepon &= ~KEEP_SEND; /* no writing anymore either */ 830 } 831 832 return CURLE_OK; 833 } 834 835 static CURLcode done_sending(struct connectdata *conn, 836 struct SingleRequest *k) 837 { 838 k->keepon &= ~KEEP_SEND; /* we're done writing */ 839 840 Curl_http2_done_sending(conn); 841 842 if(conn->bits.rewindaftersend) { 843 CURLcode result = Curl_readrewind(conn); 844 if(result) 845 return result; 846 } 847 return CURLE_OK; 848 } 849 850 851 /* 852 * Send data to upload to the server, when the socket is writable. 853 */ 854 static CURLcode readwrite_upload(struct Curl_easy *data, 855 struct connectdata *conn, 856 struct SingleRequest *k, 857 int *didwhat) 858 { 859 ssize_t i, si; 860 ssize_t bytes_written; 861 CURLcode result; 862 ssize_t nread; /* number of bytes read */ 863 bool sending_http_headers = FALSE; 864 865 if((k->bytecount == 0) && (k->writebytecount == 0)) 866 Curl_pgrsTime(data, TIMER_STARTTRANSFER); 867 868 *didwhat |= KEEP_SEND; 869 870 do { 871 872 /* only read more data if there's no upload data already 873 present in the upload buffer */ 874 if(0 == data->req.upload_present) { 875 /* init the "upload from here" pointer */ 876 data->req.upload_fromhere = k->uploadbuf; 877 878 if(!k->upload_done) { 879 /* HTTP pollution, this should be written nicer to become more 880 protocol agnostic. */ 881 int fillcount; 882 struct HTTP *http = data->req.protop; 883 884 if((k->exp100 == EXP100_SENDING_REQUEST) && 885 (http->sending == HTTPSEND_BODY)) { 886 /* If this call is to send body data, we must take some action: 887 We have sent off the full HTTP 1.1 request, and we shall now 888 go into the Expect: 100 state and await such a header */ 889 k->exp100 = EXP100_AWAITING_CONTINUE; /* wait for the header */ 890 k->keepon &= ~KEEP_SEND; /* disable writing */ 891 k->start100 = Curl_tvnow(); /* timeout count starts now */ 892 *didwhat &= ~KEEP_SEND; /* we didn't write anything actually */ 893 894 /* set a timeout for the multi interface */ 895 Curl_expire(data, data->set.expect_100_timeout); 896 break; 897 } 898 899 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) { 900 if(http->sending == HTTPSEND_REQUEST) 901 /* We're sending the HTTP request headers, not the data. 902 Remember that so we don't change the line endings. */ 903 sending_http_headers = TRUE; 904 else 905 sending_http_headers = FALSE; 906 } 907 908 result = Curl_fillreadbuffer(conn, BUFSIZE, &fillcount); 909 if(result) 910 return result; 911 912 nread = (ssize_t)fillcount; 913 } 914 else 915 nread = 0; /* we're done uploading/reading */ 916 917 if(!nread && (k->keepon & KEEP_SEND_PAUSE)) { 918 /* this is a paused transfer */ 919 break; 920 } 921 else if(nread<=0) { 922 result = done_sending(conn, k); 923 if(result) 924 return result; 925 break; 926 } 927 928 /* store number of bytes available for upload */ 929 data->req.upload_present = nread; 930 931 /* convert LF to CRLF if so asked */ 932 if((!sending_http_headers) && ( 933 #ifdef CURL_DO_LINEEND_CONV 934 /* always convert if we're FTPing in ASCII mode */ 935 (data->set.prefer_ascii) || 936 #endif 937 (data->set.crlf))) { 938 /* Do we need to allocate a scratch buffer? */ 939 if(!data->state.scratch) { 940 data->state.scratch = malloc(2 * BUFSIZE); 941 if(!data->state.scratch) { 942 failf(data, "Failed to alloc scratch buffer!"); 943 944 return CURLE_OUT_OF_MEMORY; 945 } 946 } 947 948 /* 949 * ASCII/EBCDIC Note: This is presumably a text (not binary) 950 * transfer so the data should already be in ASCII. 951 * That means the hex values for ASCII CR (0x0d) & LF (0x0a) 952 * must be used instead of the escape sequences \r & \n. 953 */ 954 for(i = 0, si = 0; i < nread; i++, si++) { 955 if(data->req.upload_fromhere[i] == 0x0a) { 956 data->state.scratch[si++] = 0x0d; 957 data->state.scratch[si] = 0x0a; 958 if(!data->set.crlf) { 959 /* we're here only because FTP is in ASCII mode... 960 bump infilesize for the LF we just added */ 961 if(data->state.infilesize != -1) 962 data->state.infilesize++; 963 } 964 } 965 else 966 data->state.scratch[si] = data->req.upload_fromhere[i]; 967 } 968 969 if(si != nread) { 970 /* only perform the special operation if we really did replace 971 anything */ 972 nread = si; 973 974 /* upload from the new (replaced) buffer instead */ 975 data->req.upload_fromhere = data->state.scratch; 976 977 /* set the new amount too */ 978 data->req.upload_present = nread; 979 } 980 } 981 982 #ifndef CURL_DISABLE_SMTP 983 if(conn->handler->protocol & PROTO_FAMILY_SMTP) { 984 result = Curl_smtp_escape_eob(conn, nread); 985 if(result) 986 return result; 987 } 988 #endif /* CURL_DISABLE_SMTP */ 989 } /* if 0 == data->req.upload_present */ 990 else { 991 /* We have a partial buffer left from a previous "round". Use 992 that instead of reading more data */ 993 } 994 995 /* write to socket (send away data) */ 996 result = Curl_write(conn, 997 conn->writesockfd, /* socket to send to */ 998 data->req.upload_fromhere, /* buffer pointer */ 999 data->req.upload_present, /* buffer size */ 1000 &bytes_written); /* actually sent */ 1001 1002 if(result) 1003 return result; 1004 1005 if(data->set.verbose) 1006 /* show the data before we change the pointer upload_fromhere */ 1007 Curl_debug(data, CURLINFO_DATA_OUT, data->req.upload_fromhere, 1008 (size_t)bytes_written, conn); 1009 1010 k->writebytecount += bytes_written; 1011 1012 if(k->writebytecount == data->state.infilesize) { 1013 /* we have sent all data we were supposed to */ 1014 k->upload_done = TRUE; 1015 infof(data, "We are completely uploaded and fine\n"); 1016 } 1017 1018 if(data->req.upload_present != bytes_written) { 1019 /* we only wrote a part of the buffer (if anything), deal with it! */ 1020 1021 /* store the amount of bytes left in the buffer to write */ 1022 data->req.upload_present -= bytes_written; 1023 1024 /* advance the pointer where to find the buffer when the next send 1025 is to happen */ 1026 data->req.upload_fromhere += bytes_written; 1027 } 1028 else { 1029 /* we've uploaded that buffer now */ 1030 data->req.upload_fromhere = k->uploadbuf; 1031 data->req.upload_present = 0; /* no more bytes left */ 1032 1033 if(k->upload_done) { 1034 result = done_sending(conn, k); 1035 if(result) 1036 return result; 1037 } 1038 } 1039 1040 Curl_pgrsSetUploadCounter(data, k->writebytecount); 1041 1042 } WHILE_FALSE; /* just to break out from! */ 1043 1044 return CURLE_OK; 1045 } 1046 1047 /* 1048 * Curl_readwrite() is the low-level function to be called when data is to 1049 * be read and written to/from the connection. 1050 * 1051 * return '*comeback' TRUE if we didn't properly drain the socket so this 1052 * function should get called again without select() or similar in between! 1053 */ 1054 CURLcode Curl_readwrite(struct connectdata *conn, 1055 struct Curl_easy *data, 1056 bool *done, 1057 bool *comeback) 1058 { 1059 struct SingleRequest *k = &data->req; 1060 CURLcode result; 1061 int didwhat=0; 1062 1063 curl_socket_t fd_read; 1064 curl_socket_t fd_write; 1065 int select_res = conn->cselect_bits; 1066 1067 conn->cselect_bits = 0; 1068 1069 /* only use the proper socket if the *_HOLD bit is not set simultaneously as 1070 then we are in rate limiting state in that transfer direction */ 1071 1072 if((k->keepon & KEEP_RECVBITS) == KEEP_RECV) 1073 fd_read = conn->sockfd; 1074 else 1075 fd_read = CURL_SOCKET_BAD; 1076 1077 if((k->keepon & KEEP_SENDBITS) == KEEP_SEND) 1078 fd_write = conn->writesockfd; 1079 else 1080 fd_write = CURL_SOCKET_BAD; 1081 1082 if(conn->data->state.drain) { 1083 select_res |= CURL_CSELECT_IN; 1084 DEBUGF(infof(data, "Curl_readwrite: forcibly told to drain data\n")); 1085 } 1086 1087 if(!select_res) /* Call for select()/poll() only, if read/write/error 1088 status is not known. */ 1089 select_res = Curl_socket_check(fd_read, CURL_SOCKET_BAD, fd_write, 0); 1090 1091 if(select_res == CURL_CSELECT_ERR) { 1092 failf(data, "select/poll returned error"); 1093 return CURLE_SEND_ERROR; 1094 } 1095 1096 /* We go ahead and do a read if we have a readable socket or if 1097 the stream was rewound (in which case we have data in a 1098 buffer) */ 1099 if((k->keepon & KEEP_RECV) && 1100 ((select_res & CURL_CSELECT_IN) || conn->bits.stream_was_rewound)) { 1101 1102 result = readwrite_data(data, conn, k, &didwhat, done, comeback); 1103 if(result || *done) 1104 return result; 1105 } 1106 1107 /* If we still have writing to do, we check if we have a writable socket. */ 1108 if((k->keepon & KEEP_SEND) && (select_res & CURL_CSELECT_OUT)) { 1109 /* write */ 1110 1111 result = readwrite_upload(data, conn, k, &didwhat); 1112 if(result) 1113 return result; 1114 } 1115 1116 k->now = Curl_tvnow(); 1117 if(didwhat) { 1118 /* Update read/write counters */ 1119 if(k->bytecountp) 1120 *k->bytecountp = k->bytecount; /* read count */ 1121 if(k->writebytecountp) 1122 *k->writebytecountp = k->writebytecount; /* write count */ 1123 } 1124 else { 1125 /* no read no write, this is a timeout? */ 1126 if(k->exp100 == EXP100_AWAITING_CONTINUE) { 1127 /* This should allow some time for the header to arrive, but only a 1128 very short time as otherwise it'll be too much wasted time too 1129 often. */ 1130 1131 /* Quoting RFC2616, section "8.2.3 Use of the 100 (Continue) Status": 1132 1133 Therefore, when a client sends this header field to an origin server 1134 (possibly via a proxy) from which it has never seen a 100 (Continue) 1135 status, the client SHOULD NOT wait for an indefinite period before 1136 sending the request body. 1137 1138 */ 1139 1140 time_t ms = Curl_tvdiff(k->now, k->start100); 1141 if(ms >= data->set.expect_100_timeout) { 1142 /* we've waited long enough, continue anyway */ 1143 k->exp100 = EXP100_SEND_DATA; 1144 k->keepon |= KEEP_SEND; 1145 infof(data, "Done waiting for 100-continue\n"); 1146 } 1147 } 1148 } 1149 1150 if(Curl_pgrsUpdate(conn)) 1151 result = CURLE_ABORTED_BY_CALLBACK; 1152 else 1153 result = Curl_speedcheck(data, k->now); 1154 if(result) 1155 return result; 1156 1157 if(k->keepon) { 1158 if(0 > Curl_timeleft(data, &k->now, FALSE)) { 1159 if(k->size != -1) { 1160 failf(data, "Operation timed out after %ld milliseconds with %" 1161 CURL_FORMAT_CURL_OFF_T " out of %" 1162 CURL_FORMAT_CURL_OFF_T " bytes received", 1163 Curl_tvdiff(k->now, data->progress.t_startsingle), k->bytecount, 1164 k->size); 1165 } 1166 else { 1167 failf(data, "Operation timed out after %ld milliseconds with %" 1168 CURL_FORMAT_CURL_OFF_T " bytes received", 1169 Curl_tvdiff(k->now, data->progress.t_startsingle), k->bytecount); 1170 } 1171 return CURLE_OPERATION_TIMEDOUT; 1172 } 1173 } 1174 else { 1175 /* 1176 * The transfer has been performed. Just make some general checks before 1177 * returning. 1178 */ 1179 1180 if(!(data->set.opt_no_body) && (k->size != -1) && 1181 (k->bytecount != k->size) && 1182 #ifdef CURL_DO_LINEEND_CONV 1183 /* Most FTP servers don't adjust their file SIZE response for CRLFs, 1184 so we'll check to see if the discrepancy can be explained 1185 by the number of CRLFs we've changed to LFs. 1186 */ 1187 (k->bytecount != (k->size + data->state.crlf_conversions)) && 1188 #endif /* CURL_DO_LINEEND_CONV */ 1189 !data->req.newurl) { 1190 failf(data, "transfer closed with %" CURL_FORMAT_CURL_OFF_T 1191 " bytes remaining to read", 1192 k->size - k->bytecount); 1193 return CURLE_PARTIAL_FILE; 1194 } 1195 else if(!(data->set.opt_no_body) && 1196 k->chunk && 1197 (conn->chunk.state != CHUNK_STOP)) { 1198 /* 1199 * In chunked mode, return an error if the connection is closed prior to 1200 * the empty (terminating) chunk is read. 1201 * 1202 * The condition above used to check for 1203 * conn->proto.http->chunk.datasize != 0 which is true after reading 1204 * *any* chunk, not just the empty chunk. 1205 * 1206 */ 1207 failf(data, "transfer closed with outstanding read data remaining"); 1208 return CURLE_PARTIAL_FILE; 1209 } 1210 if(Curl_pgrsUpdate(conn)) 1211 return CURLE_ABORTED_BY_CALLBACK; 1212 } 1213 1214 /* Now update the "done" boolean we return */ 1215 *done = (0 == (k->keepon&(KEEP_RECV|KEEP_SEND| 1216 KEEP_RECV_PAUSE|KEEP_SEND_PAUSE))) ? TRUE : FALSE; 1217 1218 return CURLE_OK; 1219 } 1220 1221 /* 1222 * Curl_single_getsock() gets called by the multi interface code when the app 1223 * has requested to get the sockets for the current connection. This function 1224 * will then be called once for every connection that the multi interface 1225 * keeps track of. This function will only be called for connections that are 1226 * in the proper state to have this information available. 1227 */ 1228 int Curl_single_getsock(const struct connectdata *conn, 1229 curl_socket_t *sock, /* points to numsocks number 1230 of sockets */ 1231 int numsocks) 1232 { 1233 const struct Curl_easy *data = conn->data; 1234 int bitmap = GETSOCK_BLANK; 1235 unsigned sockindex = 0; 1236 1237 if(conn->handler->perform_getsock) 1238 return conn->handler->perform_getsock(conn, sock, numsocks); 1239 1240 if(numsocks < 2) 1241 /* simple check but we might need two slots */ 1242 return GETSOCK_BLANK; 1243 1244 /* don't include HOLD and PAUSE connections */ 1245 if((data->req.keepon & KEEP_RECVBITS) == KEEP_RECV) { 1246 1247 DEBUGASSERT(conn->sockfd != CURL_SOCKET_BAD); 1248 1249 bitmap |= GETSOCK_READSOCK(sockindex); 1250 sock[sockindex] = conn->sockfd; 1251 } 1252 1253 /* don't include HOLD and PAUSE connections */ 1254 if((data->req.keepon & KEEP_SENDBITS) == KEEP_SEND) { 1255 1256 if((conn->sockfd != conn->writesockfd) || 1257 bitmap == GETSOCK_BLANK) { 1258 /* only if they are not the same socket and we have a readable 1259 one, we increase index */ 1260 if(bitmap != GETSOCK_BLANK) 1261 sockindex++; /* increase index if we need two entries */ 1262 1263 DEBUGASSERT(conn->writesockfd != CURL_SOCKET_BAD); 1264 1265 sock[sockindex] = conn->writesockfd; 1266 } 1267 1268 bitmap |= GETSOCK_WRITESOCK(sockindex); 1269 } 1270 1271 return bitmap; 1272 } 1273 1274 /* Curl_init_CONNECT() gets called each time the handle switches to CONNECT 1275 which means this gets called once for each subsequent redirect etc */ 1276 void Curl_init_CONNECT(struct Curl_easy *data) 1277 { 1278 data->state.fread_func = data->set.fread_func_set; 1279 data->state.in = data->set.in_set; 1280 } 1281 1282 /* 1283 * Curl_pretransfer() is called immediately before a transfer starts, and only 1284 * once for one transfer no matter if it has redirects or do multi-pass 1285 * authentication etc. 1286 */ 1287 CURLcode Curl_pretransfer(struct Curl_easy *data) 1288 { 1289 CURLcode result; 1290 if(!data->change.url) { 1291 /* we can't do anything without URL */ 1292 failf(data, "No URL set!"); 1293 return CURLE_URL_MALFORMAT; 1294 } 1295 1296 /* Init the SSL session ID cache here. We do it here since we want to do it 1297 after the *_setopt() calls (that could specify the size of the cache) but 1298 before any transfer takes place. */ 1299 result = Curl_ssl_initsessions(data, data->set.general_ssl.max_ssl_sessions); 1300 if(result) 1301 return result; 1302 1303 data->set.followlocation=0; /* reset the location-follow counter */ 1304 data->state.this_is_a_follow = FALSE; /* reset this */ 1305 data->state.errorbuf = FALSE; /* no error has occurred */ 1306 data->state.httpversion = 0; /* don't assume any particular server version */ 1307 1308 data->state.authproblem = FALSE; 1309 data->state.authhost.want = data->set.httpauth; 1310 data->state.authproxy.want = data->set.proxyauth; 1311 Curl_safefree(data->info.wouldredirect); 1312 data->info.wouldredirect = NULL; 1313 1314 if(data->set.httpreq == HTTPREQ_PUT) 1315 data->state.infilesize = data->set.filesize; 1316 else 1317 data->state.infilesize = data->set.postfieldsize; 1318 1319 /* If there is a list of cookie files to read, do it now! */ 1320 if(data->change.cookielist) 1321 Curl_cookie_loadfiles(data); 1322 1323 /* If there is a list of host pairs to deal with */ 1324 if(data->change.resolve) 1325 result = Curl_loadhostpairs(data); 1326 1327 if(!result) { 1328 /* Allow data->set.use_port to set which port to use. This needs to be 1329 * disabled for example when we follow Location: headers to URLs using 1330 * different ports! */ 1331 data->state.allow_port = TRUE; 1332 1333 #if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL) 1334 /************************************************************* 1335 * Tell signal handler to ignore SIGPIPE 1336 *************************************************************/ 1337 if(!data->set.no_signal) 1338 data->state.prev_signal = signal(SIGPIPE, SIG_IGN); 1339 #endif 1340 1341 Curl_initinfo(data); /* reset session-specific information "variables" */ 1342 Curl_pgrsResetTimesSizes(data); 1343 Curl_pgrsStartNow(data); 1344 1345 if(data->set.timeout) 1346 Curl_expire(data, data->set.timeout); 1347 1348 if(data->set.connecttimeout) 1349 Curl_expire(data, data->set.connecttimeout); 1350 1351 /* In case the handle is re-used and an authentication method was picked 1352 in the session we need to make sure we only use the one(s) we now 1353 consider to be fine */ 1354 data->state.authhost.picked &= data->state.authhost.want; 1355 data->state.authproxy.picked &= data->state.authproxy.want; 1356 1357 if(data->set.wildcardmatch) { 1358 struct WildcardData *wc = &data->wildcard; 1359 if(!wc->filelist) { 1360 result = Curl_wildcard_init(wc); /* init wildcard structures */ 1361 if(result) 1362 return CURLE_OUT_OF_MEMORY; 1363 } 1364 } 1365 1366 } 1367 1368 return result; 1369 } 1370 1371 /* 1372 * Curl_posttransfer() is called immediately after a transfer ends 1373 */ 1374 CURLcode Curl_posttransfer(struct Curl_easy *data) 1375 { 1376 #if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL) 1377 /* restore the signal handler for SIGPIPE before we get back */ 1378 if(!data->set.no_signal) 1379 signal(SIGPIPE, data->state.prev_signal); 1380 #else 1381 (void)data; /* unused parameter */ 1382 #endif 1383 1384 return CURLE_OK; 1385 } 1386 1387 #ifndef CURL_DISABLE_HTTP 1388 /* 1389 * strlen_url() returns the length of the given URL if the spaces within the 1390 * URL were properly URL encoded. 1391 */ 1392 static size_t strlen_url(const char *url) 1393 { 1394 const unsigned char *ptr; 1395 size_t newlen=0; 1396 bool left=TRUE; /* left side of the ? */ 1397 1398 for(ptr=(unsigned char *)url; *ptr; ptr++) { 1399 switch(*ptr) { 1400 case '?': 1401 left=FALSE; 1402 /* fall through */ 1403 default: 1404 if(*ptr >= 0x80) 1405 newlen += 2; 1406 newlen++; 1407 break; 1408 case ' ': 1409 if(left) 1410 newlen+=3; 1411 else 1412 newlen++; 1413 break; 1414 } 1415 } 1416 return newlen; 1417 } 1418 1419 /* strcpy_url() copies a url to a output buffer and URL-encodes the spaces in 1420 * the source URL accordingly. 1421 */ 1422 static void strcpy_url(char *output, const char *url) 1423 { 1424 /* we must add this with whitespace-replacing */ 1425 bool left=TRUE; 1426 const unsigned char *iptr; 1427 char *optr = output; 1428 for(iptr = (unsigned char *)url; /* read from here */ 1429 *iptr; /* until zero byte */ 1430 iptr++) { 1431 switch(*iptr) { 1432 case '?': 1433 left=FALSE; 1434 /* fall through */ 1435 default: 1436 if(*iptr >= 0x80) { 1437 snprintf(optr, 4, "%%%02x", *iptr); 1438 optr += 3; 1439 } 1440 else 1441 *optr++=*iptr; 1442 break; 1443 case ' ': 1444 if(left) { 1445 *optr++='%'; /* add a '%' */ 1446 *optr++='2'; /* add a '2' */ 1447 *optr++='0'; /* add a '0' */ 1448 } 1449 else 1450 *optr++='+'; /* add a '+' here */ 1451 break; 1452 } 1453 } 1454 *optr=0; /* zero terminate output buffer */ 1455 1456 } 1457 1458 /* 1459 * Returns true if the given URL is absolute (as opposed to relative) 1460 */ 1461 static bool is_absolute_url(const char *url) 1462 { 1463 char prot[16]; /* URL protocol string storage */ 1464 char letter; /* used for a silly sscanf */ 1465 1466 return (2 == sscanf(url, "%15[^?&/:]://%c", prot, &letter)) ? TRUE : FALSE; 1467 } 1468 1469 /* 1470 * Concatenate a relative URL to a base URL making it absolute. 1471 * URL-encodes any spaces. 1472 * The returned pointer must be freed by the caller unless NULL 1473 * (returns NULL on out of memory). 1474 */ 1475 static char *concat_url(const char *base, const char *relurl) 1476 { 1477 /*** 1478 TRY to append this new path to the old URL 1479 to the right of the host part. Oh crap, this is doomed to cause 1480 problems in the future... 1481 */ 1482 char *newest; 1483 char *protsep; 1484 char *pathsep; 1485 size_t newlen; 1486 1487 const char *useurl = relurl; 1488 size_t urllen; 1489 1490 /* we must make our own copy of the URL to play with, as it may 1491 point to read-only data */ 1492 char *url_clone=strdup(base); 1493 1494 if(!url_clone) 1495 return NULL; /* skip out of this NOW */ 1496 1497 /* protsep points to the start of the host name */ 1498 protsep=strstr(url_clone, "//"); 1499 if(!protsep) 1500 protsep=url_clone; 1501 else 1502 protsep+=2; /* pass the slashes */ 1503 1504 if('/' != relurl[0]) { 1505 int level=0; 1506 1507 /* First we need to find out if there's a ?-letter in the URL, 1508 and cut it and the right-side of that off */ 1509 pathsep = strchr(protsep, '?'); 1510 if(pathsep) 1511 *pathsep=0; 1512 1513 /* we have a relative path to append to the last slash if there's one 1514 available, or if the new URL is just a query string (starts with a 1515 '?') we append the new one at the end of the entire currently worked 1516 out URL */ 1517 if(useurl[0] != '?') { 1518 pathsep = strrchr(protsep, '/'); 1519 if(pathsep) 1520 *pathsep=0; 1521 } 1522 1523 /* Check if there's any slash after the host name, and if so, remember 1524 that position instead */ 1525 pathsep = strchr(protsep, '/'); 1526 if(pathsep) 1527 protsep = pathsep+1; 1528 else 1529 protsep = NULL; 1530 1531 /* now deal with one "./" or any amount of "../" in the newurl 1532 and act accordingly */ 1533 1534 if((useurl[0] == '.') && (useurl[1] == '/')) 1535 useurl+=2; /* just skip the "./" */ 1536 1537 while((useurl[0] == '.') && 1538 (useurl[1] == '.') && 1539 (useurl[2] == '/')) { 1540 level++; 1541 useurl+=3; /* pass the "../" */ 1542 } 1543 1544 if(protsep) { 1545 while(level--) { 1546 /* cut off one more level from the right of the original URL */ 1547 pathsep = strrchr(protsep, '/'); 1548 if(pathsep) 1549 *pathsep=0; 1550 else { 1551 *protsep=0; 1552 break; 1553 } 1554 } 1555 } 1556 } 1557 else { 1558 /* We got a new absolute path for this server */ 1559 1560 if((relurl[0] == '/') && (relurl[1] == '/')) { 1561 /* the new URL starts with //, just keep the protocol part from the 1562 original one */ 1563 *protsep=0; 1564 useurl = &relurl[2]; /* we keep the slashes from the original, so we 1565 skip the new ones */ 1566 } 1567 else { 1568 /* cut off the original URL from the first slash, or deal with URLs 1569 without slash */ 1570 pathsep = strchr(protsep, '/'); 1571 if(pathsep) { 1572 /* When people use badly formatted URLs, such as 1573 "http://www.url.com?dir=/home/daniel" we must not use the first 1574 slash, if there's a ?-letter before it! */ 1575 char *sep = strchr(protsep, '?'); 1576 if(sep && (sep < pathsep)) 1577 pathsep = sep; 1578 *pathsep=0; 1579 } 1580 else { 1581 /* There was no slash. Now, since we might be operating on a badly 1582 formatted URL, such as "http://www.url.com?id=2380" which doesn't 1583 use a slash separator as it is supposed to, we need to check for a 1584 ?-letter as well! */ 1585 pathsep = strchr(protsep, '?'); 1586 if(pathsep) 1587 *pathsep=0; 1588 } 1589 } 1590 } 1591 1592 /* If the new part contains a space, this is a mighty stupid redirect 1593 but we still make an effort to do "right". To the left of a '?' 1594 letter we replace each space with %20 while it is replaced with '+' 1595 on the right side of the '?' letter. 1596 */ 1597 newlen = strlen_url(useurl); 1598 1599 urllen = strlen(url_clone); 1600 1601 newest = malloc(urllen + 1 + /* possible slash */ 1602 newlen + 1 /* zero byte */); 1603 1604 if(!newest) { 1605 free(url_clone); /* don't leak this */ 1606 return NULL; 1607 } 1608 1609 /* copy over the root url part */ 1610 memcpy(newest, url_clone, urllen); 1611 1612 /* check if we need to append a slash */ 1613 if(('/' == useurl[0]) || (protsep && !*protsep) || ('?' == useurl[0])) 1614 ; 1615 else 1616 newest[urllen++]='/'; 1617 1618 /* then append the new piece on the right side */ 1619 strcpy_url(&newest[urllen], useurl); 1620 1621 free(url_clone); 1622 1623 return newest; 1624 } 1625 #endif /* CURL_DISABLE_HTTP */ 1626 1627 /* 1628 * Curl_follow() handles the URL redirect magic. Pass in the 'newurl' string 1629 * as given by the remote server and set up the new URL to request. 1630 */ 1631 CURLcode Curl_follow(struct Curl_easy *data, 1632 char *newurl, /* this 'newurl' is the Location: string, 1633 and it must be malloc()ed before passed 1634 here */ 1635 followtype type) /* see transfer.h */ 1636 { 1637 #ifdef CURL_DISABLE_HTTP 1638 (void)data; 1639 (void)newurl; 1640 (void)type; 1641 /* Location: following will not happen when HTTP is disabled */ 1642 return CURLE_TOO_MANY_REDIRECTS; 1643 #else 1644 1645 /* Location: redirect */ 1646 bool disallowport = FALSE; 1647 1648 if(type == FOLLOW_REDIR) { 1649 if((data->set.maxredirs != -1) && 1650 (data->set.followlocation >= data->set.maxredirs)) { 1651 failf(data, "Maximum (%ld) redirects followed", data->set.maxredirs); 1652 return CURLE_TOO_MANY_REDIRECTS; 1653 } 1654 1655 /* mark the next request as a followed location: */ 1656 data->state.this_is_a_follow = TRUE; 1657 1658 data->set.followlocation++; /* count location-followers */ 1659 1660 if(data->set.http_auto_referer) { 1661 /* We are asked to automatically set the previous URL as the referer 1662 when we get the next URL. We pick the ->url field, which may or may 1663 not be 100% correct */ 1664 1665 if(data->change.referer_alloc) { 1666 Curl_safefree(data->change.referer); 1667 data->change.referer_alloc = FALSE; 1668 } 1669 1670 data->change.referer = strdup(data->change.url); 1671 if(!data->change.referer) 1672 return CURLE_OUT_OF_MEMORY; 1673 data->change.referer_alloc = TRUE; /* yes, free this later */ 1674 } 1675 } 1676 1677 if(!is_absolute_url(newurl)) { 1678 /*** 1679 *DANG* this is an RFC 2068 violation. The URL is supposed 1680 to be absolute and this doesn't seem to be that! 1681 */ 1682 char *absolute = concat_url(data->change.url, newurl); 1683 if(!absolute) 1684 return CURLE_OUT_OF_MEMORY; 1685 free(newurl); 1686 newurl = absolute; 1687 } 1688 else { 1689 /* The new URL MAY contain space or high byte values, that means a mighty 1690 stupid redirect URL but we still make an effort to do "right". */ 1691 char *newest; 1692 size_t newlen = strlen_url(newurl); 1693 1694 /* This is an absolute URL, don't allow the custom port number */ 1695 disallowport = TRUE; 1696 1697 newest = malloc(newlen+1); /* get memory for this */ 1698 if(!newest) 1699 return CURLE_OUT_OF_MEMORY; 1700 strcpy_url(newest, newurl); /* create a space-free URL */ 1701 1702 free(newurl); /* that was no good */ 1703 newurl = newest; /* use this instead now */ 1704 1705 } 1706 1707 if(type == FOLLOW_FAKE) { 1708 /* we're only figuring out the new url if we would've followed locations 1709 but now we're done so we can get out! */ 1710 data->info.wouldredirect = newurl; 1711 return CURLE_OK; 1712 } 1713 1714 if(disallowport) 1715 data->state.allow_port = FALSE; 1716 1717 if(data->change.url_alloc) { 1718 Curl_safefree(data->change.url); 1719 data->change.url_alloc = FALSE; 1720 } 1721 1722 data->change.url = newurl; 1723 data->change.url_alloc = TRUE; 1724 newurl = NULL; /* don't free! */ 1725 1726 infof(data, "Issue another request to this URL: '%s'\n", data->change.url); 1727 1728 /* 1729 * We get here when the HTTP code is 300-399 (and 401). We need to perform 1730 * differently based on exactly what return code there was. 1731 * 1732 * News from 7.10.6: we can also get here on a 401 or 407, in case we act on 1733 * a HTTP (proxy-) authentication scheme other than Basic. 1734 */ 1735 switch(data->info.httpcode) { 1736 /* 401 - Act on a WWW-Authenticate, we keep on moving and do the 1737 Authorization: XXXX header in the HTTP request code snippet */ 1738 /* 407 - Act on a Proxy-Authenticate, we keep on moving and do the 1739 Proxy-Authorization: XXXX header in the HTTP request code snippet */ 1740 /* 300 - Multiple Choices */ 1741 /* 306 - Not used */ 1742 /* 307 - Temporary Redirect */ 1743 default: /* for all above (and the unknown ones) */ 1744 /* Some codes are explicitly mentioned since I've checked RFC2616 and they 1745 * seem to be OK to POST to. 1746 */ 1747 break; 1748 case 301: /* Moved Permanently */ 1749 /* (quote from RFC7231, section 6.4.2) 1750 * 1751 * Note: For historical reasons, a user agent MAY change the request 1752 * method from POST to GET for the subsequent request. If this 1753 * behavior is undesired, the 307 (Temporary Redirect) status code 1754 * can be used instead. 1755 * 1756 * ---- 1757 * 1758 * Many webservers expect this, so these servers often answers to a POST 1759 * request with an error page. To be sure that libcurl gets the page that 1760 * most user agents would get, libcurl has to force GET. 1761 * 1762 * This behaviour is forbidden by RFC1945 and the obsolete RFC2616, and 1763 * can be overridden with CURLOPT_POSTREDIR. 1764 */ 1765 if((data->set.httpreq == HTTPREQ_POST 1766 || data->set.httpreq == HTTPREQ_POST_FORM) 1767 && !(data->set.keep_post & CURL_REDIR_POST_301)) { 1768 infof(data, "Switch from POST to GET\n"); 1769 data->set.httpreq = HTTPREQ_GET; 1770 } 1771 break; 1772 case 302: /* Found */ 1773 /* (quote from RFC7231, section 6.4.3) 1774 * 1775 * Note: For historical reasons, a user agent MAY change the request 1776 * method from POST to GET for the subsequent request. If this 1777 * behavior is undesired, the 307 (Temporary Redirect) status code 1778 * can be used instead. 1779 * 1780 * ---- 1781 * 1782 * Many webservers expect this, so these servers often answers to a POST 1783 * request with an error page. To be sure that libcurl gets the page that 1784 * most user agents would get, libcurl has to force GET. 1785 * 1786 * This behaviour is forbidden by RFC1945 and the obsolete RFC2616, and 1787 * can be overridden with CURLOPT_POSTREDIR. 1788 */ 1789 if((data->set.httpreq == HTTPREQ_POST 1790 || data->set.httpreq == HTTPREQ_POST_FORM) 1791 && !(data->set.keep_post & CURL_REDIR_POST_302)) { 1792 infof(data, "Switch from POST to GET\n"); 1793 data->set.httpreq = HTTPREQ_GET; 1794 } 1795 break; 1796 1797 case 303: /* See Other */ 1798 /* Disable both types of POSTs, unless the user explicitely 1799 asks for POST after POST */ 1800 if(data->set.httpreq != HTTPREQ_GET 1801 && !(data->set.keep_post & CURL_REDIR_POST_303)) { 1802 data->set.httpreq = HTTPREQ_GET; /* enforce GET request */ 1803 infof(data, "Disables POST, goes with %s\n", 1804 data->set.opt_no_body?"HEAD":"GET"); 1805 } 1806 break; 1807 case 304: /* Not Modified */ 1808 /* 304 means we did a conditional request and it was "Not modified". 1809 * We shouldn't get any Location: header in this response! 1810 */ 1811 break; 1812 case 305: /* Use Proxy */ 1813 /* (quote from RFC2616, section 10.3.6): 1814 * "The requested resource MUST be accessed through the proxy given 1815 * by the Location field. The Location field gives the URI of the 1816 * proxy. The recipient is expected to repeat this single request 1817 * via the proxy. 305 responses MUST only be generated by origin 1818 * servers." 1819 */ 1820 break; 1821 } 1822 Curl_pgrsTime(data, TIMER_REDIRECT); 1823 Curl_pgrsResetTimesSizes(data); 1824 1825 return CURLE_OK; 1826 #endif /* CURL_DISABLE_HTTP */ 1827 } 1828 1829 /* Returns CURLE_OK *and* sets '*url' if a request retry is wanted. 1830 1831 NOTE: that the *url is malloc()ed. */ 1832 CURLcode Curl_retry_request(struct connectdata *conn, 1833 char **url) 1834 { 1835 struct Curl_easy *data = conn->data; 1836 1837 *url = NULL; 1838 1839 /* if we're talking upload, we can't do the checks below, unless the protocol 1840 is HTTP as when uploading over HTTP we will still get a response */ 1841 if(data->set.upload && 1842 !(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP))) 1843 return CURLE_OK; 1844 1845 if((data->req.bytecount + data->req.headerbytecount == 0) && 1846 conn->bits.reuse && 1847 (data->set.rtspreq != RTSPREQ_RECEIVE)) { 1848 /* We didn't get a single byte when we attempted to re-use a 1849 connection. This might happen if the connection was left alive when we 1850 were done using it before, but that was closed when we wanted to use it 1851 again. Bad luck. Retry the same request on a fresh connect! */ 1852 infof(conn->data, "Connection died, retrying a fresh connect\n"); 1853 *url = strdup(conn->data->change.url); 1854 if(!*url) 1855 return CURLE_OUT_OF_MEMORY; 1856 1857 connclose(conn, "retry"); /* close this connection */ 1858 conn->bits.retry = TRUE; /* mark this as a connection we're about 1859 to retry. Marking it this way should 1860 prevent i.e HTTP transfers to return 1861 error just because nothing has been 1862 transferred! */ 1863 1864 1865 if(conn->handler->protocol&PROTO_FAMILY_HTTP) { 1866 struct HTTP *http = data->req.protop; 1867 if(http->writebytecount) 1868 return Curl_readrewind(conn); 1869 } 1870 } 1871 return CURLE_OK; 1872 } 1873 1874 /* 1875 * Curl_setup_transfer() is called to setup some basic properties for the 1876 * upcoming transfer. 1877 */ 1878 void 1879 Curl_setup_transfer( 1880 struct connectdata *conn, /* connection data */ 1881 int sockindex, /* socket index to read from or -1 */ 1882 curl_off_t size, /* -1 if unknown at this point */ 1883 bool getheader, /* TRUE if header parsing is wanted */ 1884 curl_off_t *bytecountp, /* return number of bytes read or NULL */ 1885 int writesockindex, /* socket index to write to, it may very well be 1886 the same we read from. -1 disables */ 1887 curl_off_t *writecountp /* return number of bytes written or NULL */ 1888 ) 1889 { 1890 struct Curl_easy *data; 1891 struct SingleRequest *k; 1892 1893 DEBUGASSERT(conn != NULL); 1894 1895 data = conn->data; 1896 k = &data->req; 1897 1898 DEBUGASSERT((sockindex <= 1) && (sockindex >= -1)); 1899 1900 /* now copy all input parameters */ 1901 conn->sockfd = sockindex == -1 ? 1902 CURL_SOCKET_BAD : conn->sock[sockindex]; 1903 conn->writesockfd = writesockindex == -1 ? 1904 CURL_SOCKET_BAD:conn->sock[writesockindex]; 1905 k->getheader = getheader; 1906 1907 k->size = size; 1908 k->bytecountp = bytecountp; 1909 k->writebytecountp = writecountp; 1910 1911 /* The code sequence below is placed in this function just because all 1912 necessary input is not always known in do_complete() as this function may 1913 be called after that */ 1914 1915 if(!k->getheader) { 1916 k->header = FALSE; 1917 if(size > 0) 1918 Curl_pgrsSetDownloadSize(data, size); 1919 } 1920 /* we want header and/or body, if neither then don't do this! */ 1921 if(k->getheader || !data->set.opt_no_body) { 1922 1923 if(conn->sockfd != CURL_SOCKET_BAD) 1924 k->keepon |= KEEP_RECV; 1925 1926 if(conn->writesockfd != CURL_SOCKET_BAD) { 1927 struct HTTP *http = data->req.protop; 1928 /* HTTP 1.1 magic: 1929 1930 Even if we require a 100-return code before uploading data, we might 1931 need to write data before that since the REQUEST may not have been 1932 finished sent off just yet. 1933 1934 Thus, we must check if the request has been sent before we set the 1935 state info where we wait for the 100-return code 1936 */ 1937 if((data->state.expect100header) && 1938 (conn->handler->protocol&PROTO_FAMILY_HTTP) && 1939 (http->sending == HTTPSEND_BODY)) { 1940 /* wait with write until we either got 100-continue or a timeout */ 1941 k->exp100 = EXP100_AWAITING_CONTINUE; 1942 k->start100 = Curl_tvnow(); 1943 1944 /* Set a timeout for the multi interface. Add the inaccuracy margin so 1945 that we don't fire slightly too early and get denied to run. */ 1946 Curl_expire(data, data->set.expect_100_timeout); 1947 } 1948 else { 1949 if(data->state.expect100header) 1950 /* when we've sent off the rest of the headers, we must await a 1951 100-continue but first finish sending the request */ 1952 k->exp100 = EXP100_SENDING_REQUEST; 1953 1954 /* enable the write bit when we're not waiting for continue */ 1955 k->keepon |= KEEP_SEND; 1956 } 1957 } /* if(conn->writesockfd != CURL_SOCKET_BAD) */ 1958 } /* if(k->getheader || !data->set.opt_no_body) */ 1959 1960 } 1961
