1 /***************************************************************************** 2 * 3 * This example source code introduces a c library buffered I/O interface to 4 * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(), 5 * rewind(). Supported functions have identical prototypes to their normal c 6 * lib namesakes and are preceaded by url_ . 7 * 8 * Using this code you can replace your program's fopen() with url_fopen() 9 * and fread() with url_fread() and it become possible to read remote streams 10 * instead of (only) local files. Local files (ie those that can be directly 11 * fopened) will drop back to using the underlying clib implementations 12 * 13 * See the main() function at the bottom that shows an app that retrieves from 14 * a specified url using fgets() and fread() and saves as two output files. 15 * 16 * Copyright (c) 2003, 2017 Simtec Electronics 17 * 18 * Re-implemented by Vincent Sanders <vince (at) kyllikki.org> with extensive 19 * reference to original curl example code 20 * 21 * Redistribution and use in source and binary forms, with or without 22 * modification, are permitted provided that the following conditions 23 * are met: 24 * 1. Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * 2. Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in the 28 * documentation and/or other materials provided with the distribution. 29 * 3. The name of the author may not be used to endorse or promote products 30 * derived from this software without specific prior written permission. 31 * 32 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 33 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 34 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 35 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 36 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 37 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 41 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 42 * 43 * This example requires libcurl 7.9.7 or later. 44 */ 45 /* <DESC> 46 * implements an fopen() abstraction allowing reading from URLs 47 * </DESC> 48 */ 49 50 #include <stdio.h> 51 #include <string.h> 52 #ifndef WIN32 53 # include <sys/time.h> 54 #endif 55 #include <stdlib.h> 56 #include <errno.h> 57 58 #include <curl/curl.h> 59 60 enum fcurl_type_e { 61 CFTYPE_NONE = 0, 62 CFTYPE_FILE = 1, 63 CFTYPE_CURL = 2 64 }; 65 66 struct fcurl_data 67 { 68 enum fcurl_type_e type; /* type of handle */ 69 union { 70 CURL *curl; 71 FILE *file; 72 } handle; /* handle */ 73 74 char *buffer; /* buffer to store cached data*/ 75 size_t buffer_len; /* currently allocated buffers length */ 76 size_t buffer_pos; /* end of data in buffer*/ 77 int still_running; /* Is background url fetch still in progress */ 78 }; 79 80 typedef struct fcurl_data URL_FILE; 81 82 /* exported functions */ 83 URL_FILE *url_fopen(const char *url, const char *operation); 84 int url_fclose(URL_FILE *file); 85 int url_feof(URL_FILE *file); 86 size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file); 87 char *url_fgets(char *ptr, size_t size, URL_FILE *file); 88 void url_rewind(URL_FILE *file); 89 90 /* we use a global one for convenience */ 91 static CURLM *multi_handle; 92 93 /* curl calls this routine to get more data */ 94 static size_t write_callback(char *buffer, 95 size_t size, 96 size_t nitems, 97 void *userp) 98 { 99 char *newbuff; 100 size_t rembuff; 101 102 URL_FILE *url = (URL_FILE *)userp; 103 size *= nitems; 104 105 rembuff = url->buffer_len - url->buffer_pos; /* remaining space in buffer */ 106 107 if(size > rembuff) { 108 /* not enough space in buffer */ 109 newbuff = realloc(url->buffer, url->buffer_len + (size - rembuff)); 110 if(newbuff == NULL) { 111 fprintf(stderr, "callback buffer grow failed\n"); 112 size = rembuff; 113 } 114 else { 115 /* realloc succeeded increase buffer size*/ 116 url->buffer_len += size - rembuff; 117 url->buffer = newbuff; 118 } 119 } 120 121 memcpy(&url->buffer[url->buffer_pos], buffer, size); 122 url->buffer_pos += size; 123 124 return size; 125 } 126 127 /* use to attempt to fill the read buffer up to requested number of bytes */ 128 static int fill_buffer(URL_FILE *file, size_t want) 129 { 130 fd_set fdread; 131 fd_set fdwrite; 132 fd_set fdexcep; 133 struct timeval timeout; 134 int rc; 135 CURLMcode mc; /* curl_multi_fdset() return code */ 136 137 /* only attempt to fill buffer if transactions still running and buffer 138 * doesn't exceed required size already 139 */ 140 if((!file->still_running) || (file->buffer_pos > want)) 141 return 0; 142 143 /* attempt to fill buffer */ 144 do { 145 int maxfd = -1; 146 long curl_timeo = -1; 147 148 FD_ZERO(&fdread); 149 FD_ZERO(&fdwrite); 150 FD_ZERO(&fdexcep); 151 152 /* set a suitable timeout to fail on */ 153 timeout.tv_sec = 60; /* 1 minute */ 154 timeout.tv_usec = 0; 155 156 curl_multi_timeout(multi_handle, &curl_timeo); 157 if(curl_timeo >= 0) { 158 timeout.tv_sec = curl_timeo / 1000; 159 if(timeout.tv_sec > 1) 160 timeout.tv_sec = 1; 161 else 162 timeout.tv_usec = (curl_timeo % 1000) * 1000; 163 } 164 165 /* get file descriptors from the transfers */ 166 mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd); 167 168 if(mc != CURLM_OK) { 169 fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc); 170 break; 171 } 172 173 /* On success the value of maxfd is guaranteed to be >= -1. We call 174 select(maxfd + 1, ...); specially in case of (maxfd == -1) there are 175 no fds ready yet so we call select(0, ...) --or Sleep() on Windows-- 176 to sleep 100ms, which is the minimum suggested value in the 177 curl_multi_fdset() doc. */ 178 179 if(maxfd == -1) { 180 #ifdef _WIN32 181 Sleep(100); 182 rc = 0; 183 #else 184 /* Portable sleep for platforms other than Windows. */ 185 struct timeval wait = { 0, 100 * 1000 }; /* 100ms */ 186 rc = select(0, NULL, NULL, NULL, &wait); 187 #endif 188 } 189 else { 190 /* Note that on some platforms 'timeout' may be modified by select(). 191 If you need access to the original value save a copy beforehand. */ 192 rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout); 193 } 194 195 switch(rc) { 196 case -1: 197 /* select error */ 198 break; 199 200 case 0: 201 default: 202 /* timeout or readable/writable sockets */ 203 curl_multi_perform(multi_handle, &file->still_running); 204 break; 205 } 206 } while(file->still_running && (file->buffer_pos < want)); 207 return 1; 208 } 209 210 /* use to remove want bytes from the front of a files buffer */ 211 static int use_buffer(URL_FILE *file, size_t want) 212 { 213 /* sort out buffer */ 214 if((file->buffer_pos - want) <= 0) { 215 /* ditch buffer - write will recreate */ 216 free(file->buffer); 217 file->buffer = NULL; 218 file->buffer_pos = 0; 219 file->buffer_len = 0; 220 } 221 else { 222 /* move rest down make it available for later */ 223 memmove(file->buffer, 224 &file->buffer[want], 225 (file->buffer_pos - want)); 226 227 file->buffer_pos -= want; 228 } 229 return 0; 230 } 231 232 URL_FILE *url_fopen(const char *url, const char *operation) 233 { 234 /* this code could check for URLs or types in the 'url' and 235 basically use the real fopen() for standard files */ 236 237 URL_FILE *file; 238 (void)operation; 239 240 file = calloc(1, sizeof(URL_FILE)); 241 if(!file) 242 return NULL; 243 244 file->handle.file = fopen(url, operation); 245 if(file->handle.file) 246 file->type = CFTYPE_FILE; /* marked as URL */ 247 248 else { 249 file->type = CFTYPE_CURL; /* marked as URL */ 250 file->handle.curl = curl_easy_init(); 251 252 curl_easy_setopt(file->handle.curl, CURLOPT_URL, url); 253 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file); 254 curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L); 255 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback); 256 257 if(!multi_handle) 258 multi_handle = curl_multi_init(); 259 260 curl_multi_add_handle(multi_handle, file->handle.curl); 261 262 /* lets start the fetch */ 263 curl_multi_perform(multi_handle, &file->still_running); 264 265 if((file->buffer_pos == 0) && (!file->still_running)) { 266 /* if still_running is 0 now, we should return NULL */ 267 268 /* make sure the easy handle is not in the multi handle anymore */ 269 curl_multi_remove_handle(multi_handle, file->handle.curl); 270 271 /* cleanup */ 272 curl_easy_cleanup(file->handle.curl); 273 274 free(file); 275 276 file = NULL; 277 } 278 } 279 return file; 280 } 281 282 int url_fclose(URL_FILE *file) 283 { 284 int ret = 0;/* default is good return */ 285 286 switch(file->type) { 287 case CFTYPE_FILE: 288 ret = fclose(file->handle.file); /* passthrough */ 289 break; 290 291 case CFTYPE_CURL: 292 /* make sure the easy handle is not in the multi handle anymore */ 293 curl_multi_remove_handle(multi_handle, file->handle.curl); 294 295 /* cleanup */ 296 curl_easy_cleanup(file->handle.curl); 297 break; 298 299 default: /* unknown or supported type - oh dear */ 300 ret = EOF; 301 errno = EBADF; 302 break; 303 } 304 305 free(file->buffer);/* free any allocated buffer space */ 306 free(file); 307 308 return ret; 309 } 310 311 int url_feof(URL_FILE *file) 312 { 313 int ret = 0; 314 315 switch(file->type) { 316 case CFTYPE_FILE: 317 ret = feof(file->handle.file); 318 break; 319 320 case CFTYPE_CURL: 321 if((file->buffer_pos == 0) && (!file->still_running)) 322 ret = 1; 323 break; 324 325 default: /* unknown or supported type - oh dear */ 326 ret = -1; 327 errno = EBADF; 328 break; 329 } 330 return ret; 331 } 332 333 size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file) 334 { 335 size_t want; 336 337 switch(file->type) { 338 case CFTYPE_FILE: 339 want = fread(ptr, size, nmemb, file->handle.file); 340 break; 341 342 case CFTYPE_CURL: 343 want = nmemb * size; 344 345 fill_buffer(file, want); 346 347 /* check if there's data in the buffer - if not fill_buffer() 348 * either errored or EOF */ 349 if(!file->buffer_pos) 350 return 0; 351 352 /* ensure only available data is considered */ 353 if(file->buffer_pos < want) 354 want = file->buffer_pos; 355 356 /* xfer data to caller */ 357 memcpy(ptr, file->buffer, want); 358 359 use_buffer(file, want); 360 361 want = want / size; /* number of items */ 362 break; 363 364 default: /* unknown or supported type - oh dear */ 365 want = 0; 366 errno = EBADF; 367 break; 368 369 } 370 return want; 371 } 372 373 char *url_fgets(char *ptr, size_t size, URL_FILE *file) 374 { 375 size_t want = size - 1;/* always need to leave room for zero termination */ 376 size_t loop; 377 378 switch(file->type) { 379 case CFTYPE_FILE: 380 ptr = fgets(ptr, (int)size, file->handle.file); 381 break; 382 383 case CFTYPE_CURL: 384 fill_buffer(file, want); 385 386 /* check if there's data in the buffer - if not fill either errored or 387 * EOF */ 388 if(!file->buffer_pos) 389 return NULL; 390 391 /* ensure only available data is considered */ 392 if(file->buffer_pos < want) 393 want = file->buffer_pos; 394 395 /*buffer contains data */ 396 /* look for newline or eof */ 397 for(loop = 0; loop < want; loop++) { 398 if(file->buffer[loop] == '\n') { 399 want = loop + 1;/* include newline */ 400 break; 401 } 402 } 403 404 /* xfer data to caller */ 405 memcpy(ptr, file->buffer, want); 406 ptr[want] = 0;/* always null terminate */ 407 408 use_buffer(file, want); 409 410 break; 411 412 default: /* unknown or supported type - oh dear */ 413 ptr = NULL; 414 errno = EBADF; 415 break; 416 } 417 418 return ptr;/*success */ 419 } 420 421 void url_rewind(URL_FILE *file) 422 { 423 switch(file->type) { 424 case CFTYPE_FILE: 425 rewind(file->handle.file); /* passthrough */ 426 break; 427 428 case CFTYPE_CURL: 429 /* halt transaction */ 430 curl_multi_remove_handle(multi_handle, file->handle.curl); 431 432 /* restart */ 433 curl_multi_add_handle(multi_handle, file->handle.curl); 434 435 /* ditch buffer - write will recreate - resets stream pos*/ 436 free(file->buffer); 437 file->buffer = NULL; 438 file->buffer_pos = 0; 439 file->buffer_len = 0; 440 441 break; 442 443 default: /* unknown or supported type - oh dear */ 444 break; 445 } 446 } 447 448 #define FGETSFILE "fgets.test" 449 #define FREADFILE "fread.test" 450 #define REWINDFILE "rewind.test" 451 452 /* Small main program to retrieve from a url using fgets and fread saving the 453 * output to two test files (note the fgets method will corrupt binary files if 454 * they contain 0 chars */ 455 int main(int argc, char *argv[]) 456 { 457 URL_FILE *handle; 458 FILE *outf; 459 460 size_t nread; 461 char buffer[256]; 462 const char *url; 463 464 if(argc < 2) 465 url = "http://192.168.7.3/testfile";/* default to testurl */ 466 else 467 url = argv[1];/* use passed url */ 468 469 /* copy from url line by line with fgets */ 470 outf = fopen(FGETSFILE, "wb+"); 471 if(!outf) { 472 perror("couldn't open fgets output file\n"); 473 return 1; 474 } 475 476 handle = url_fopen(url, "r"); 477 if(!handle) { 478 printf("couldn't url_fopen() %s\n", url); 479 fclose(outf); 480 return 2; 481 } 482 483 while(!url_feof(handle)) { 484 url_fgets(buffer, sizeof(buffer), handle); 485 fwrite(buffer, 1, strlen(buffer), outf); 486 } 487 488 url_fclose(handle); 489 490 fclose(outf); 491 492 493 /* Copy from url with fread */ 494 outf = fopen(FREADFILE, "wb+"); 495 if(!outf) { 496 perror("couldn't open fread output file\n"); 497 return 1; 498 } 499 500 handle = url_fopen("testfile", "r"); 501 if(!handle) { 502 printf("couldn't url_fopen() testfile\n"); 503 fclose(outf); 504 return 2; 505 } 506 507 do { 508 nread = url_fread(buffer, 1, sizeof(buffer), handle); 509 fwrite(buffer, 1, nread, outf); 510 } while(nread); 511 512 url_fclose(handle); 513 514 fclose(outf); 515 516 517 /* Test rewind */ 518 outf = fopen(REWINDFILE, "wb+"); 519 if(!outf) { 520 perror("couldn't open fread output file\n"); 521 return 1; 522 } 523 524 handle = url_fopen("testfile", "r"); 525 if(!handle) { 526 printf("couldn't url_fopen() testfile\n"); 527 fclose(outf); 528 return 2; 529 } 530 531 nread = url_fread(buffer, 1, sizeof(buffer), handle); 532 fwrite(buffer, 1, nread, outf); 533 url_rewind(handle); 534 535 buffer[0]='\n'; 536 fwrite(buffer, 1, 1, outf); 537 538 nread = url_fread(buffer, 1, sizeof(buffer), handle); 539 fwrite(buffer, 1, nread, outf); 540 541 url_fclose(handle); 542 543 fclose(outf); 544 545 return 0;/* all done */ 546 } 547