1 /***************************************************************************** 2 * 3 * This example source code introduces a c library buffered I/O interface to 4 * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(), 5 * rewind(). Supported functions have identical prototypes to their normal c 6 * lib namesakes and are preceaded by url_ . 7 * 8 * Using this code you can replace your program's fopen() with url_fopen() 9 * and fread() with url_fread() and it become possible to read remote streams 10 * instead of (only) local files. Local files (ie those that can be directly 11 * fopened) will drop back to using the underlying clib implementations 12 * 13 * See the main() function at the bottom that shows an app that retrieves from 14 * a specified url using fgets() and fread() and saves as two output files. 15 * 16 * Copyright (c) 2003, 2017 Simtec Electronics 17 * 18 * Re-implemented by Vincent Sanders <vince (at) kyllikki.org> with extensive 19 * reference to original curl example code 20 * 21 * Redistribution and use in source and binary forms, with or without 22 * modification, are permitted provided that the following conditions 23 * are met: 24 * 1. Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * 2. Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in the 28 * documentation and/or other materials provided with the distribution. 29 * 3. The name of the author may not be used to endorse or promote products 30 * derived from this software without specific prior written permission. 31 * 32 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 33 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 34 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 35 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 36 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 37 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 41 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 42 * 43 * This example requires libcurl 7.9.7 or later. 44 */ 45 /* <DESC> 46 * implements an fopen() abstraction allowing reading from URLs 47 * </DESC> 48 */ 49 50 #include <stdio.h> 51 #include <string.h> 52 #ifndef WIN32 53 # include <sys/time.h> 54 #endif 55 #include <stdlib.h> 56 #include <errno.h> 57 58 #include <curl/curl.h> 59 60 enum fcurl_type_e { 61 CFTYPE_NONE = 0, 62 CFTYPE_FILE = 1, 63 CFTYPE_CURL = 2 64 }; 65 66 struct fcurl_data 67 { 68 enum fcurl_type_e type; /* type of handle */ 69 union { 70 CURL *curl; 71 FILE *file; 72 } handle; /* handle */ 73 74 char *buffer; /* buffer to store cached data*/ 75 size_t buffer_len; /* currently allocated buffers length */ 76 size_t buffer_pos; /* end of data in buffer*/ 77 int still_running; /* Is background url fetch still in progress */ 78 }; 79 80 typedef struct fcurl_data URL_FILE; 81 82 /* exported functions */ 83 URL_FILE *url_fopen(const char *url, const char *operation); 84 int url_fclose(URL_FILE *file); 85 int url_feof(URL_FILE *file); 86 size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file); 87 char *url_fgets(char *ptr, size_t size, URL_FILE *file); 88 void url_rewind(URL_FILE *file); 89 90 /* we use a global one for convenience */ 91 static CURLM *multi_handle; 92 93 /* curl calls this routine to get more data */ 94 static size_t write_callback(char *buffer, 95 size_t size, 96 size_t nitems, 97 void *userp) 98 { 99 char *newbuff; 100 size_t rembuff; 101 102 URL_FILE *url = (URL_FILE *)userp; 103 size *= nitems; 104 105 rembuff = url->buffer_len - url->buffer_pos; /* remaining space in buffer */ 106 107 if(size > rembuff) { 108 /* not enough space in buffer */ 109 newbuff = realloc(url->buffer, url->buffer_len + (size - rembuff)); 110 if(newbuff == NULL) { 111 fprintf(stderr, "callback buffer grow failed\n"); 112 size = rembuff; 113 } 114 else { 115 /* realloc succeeded increase buffer size*/ 116 url->buffer_len += size - rembuff; 117 url->buffer = newbuff; 118 } 119 } 120 121 memcpy(&url->buffer[url->buffer_pos], buffer, size); 122 url->buffer_pos += size; 123 124 return size; 125 } 126 127 /* use to attempt to fill the read buffer up to requested number of bytes */ 128 static int fill_buffer(URL_FILE *file, size_t want) 129 { 130 fd_set fdread; 131 fd_set fdwrite; 132 fd_set fdexcep; 133 struct timeval timeout; 134 int rc; 135 CURLMcode mc; /* curl_multi_fdset() return code */ 136 137 /* only attempt to fill buffer if transactions still running and buffer 138 * doesn't exceed required size already 139 */ 140 if((!file->still_running) || (file->buffer_pos > want)) 141 return 0; 142 143 /* attempt to fill buffer */ 144 do { 145 int maxfd = -1; 146 long curl_timeo = -1; 147 148 FD_ZERO(&fdread); 149 FD_ZERO(&fdwrite); 150 FD_ZERO(&fdexcep); 151 152 /* set a suitable timeout to fail on */ 153 timeout.tv_sec = 60; /* 1 minute */ 154 timeout.tv_usec = 0; 155 156 curl_multi_timeout(multi_handle, &curl_timeo); 157 if(curl_timeo >= 0) { 158 timeout.tv_sec = curl_timeo / 1000; 159 if(timeout.tv_sec > 1) 160 timeout.tv_sec = 1; 161 else 162 timeout.tv_usec = (curl_timeo % 1000) * 1000; 163 } 164 165 /* get file descriptors from the transfers */ 166 mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd); 167 168 if(mc != CURLM_OK) { 169 fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc); 170 break; 171 } 172 173 /* On success the value of maxfd is guaranteed to be >= -1. We call 174 select(maxfd + 1, ...); specially in case of (maxfd == -1) there are 175 no fds ready yet so we call select(0, ...) --or Sleep() on Windows-- 176 to sleep 100ms, which is the minimum suggested value in the 177 curl_multi_fdset() doc. */ 178 179 if(maxfd == -1) { 180 #ifdef _WIN32 181 Sleep(100); 182 rc = 0; 183 #else 184 /* Portable sleep for platforms other than Windows. */ 185 struct timeval wait = { 0, 100 * 1000 }; /* 100ms */ 186 rc = select(0, NULL, NULL, NULL, &wait); 187 #endif 188 } 189 else { 190 /* Note that on some platforms 'timeout' may be modified by select(). 191 If you need access to the original value save a copy beforehand. */ 192 rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout); 193 } 194 195 switch(rc) { 196 case -1: 197 /* select error */ 198 break; 199 200 case 0: 201 default: 202 /* timeout or readable/writable sockets */ 203 curl_multi_perform(multi_handle, &file->still_running); 204 break; 205 } 206 } while(file->still_running && (file->buffer_pos < want)); 207 return 1; 208 } 209 210 /* use to remove want bytes from the front of a files buffer */ 211 static int use_buffer(URL_FILE *file, size_t want) 212 { 213 /* sort out buffer */ 214 if((file->buffer_pos - want) <= 0) { 215 /* ditch buffer - write will recreate */ 216 free(file->buffer); 217 file->buffer = NULL; 218 file->buffer_pos = 0; 219 file->buffer_len = 0; 220 } 221 else { 222 /* move rest down make it available for later */ 223 memmove(file->buffer, 224 &file->buffer[want], 225 (file->buffer_pos - want)); 226 227 file->buffer_pos -= want; 228 } 229 return 0; 230 } 231 232 URL_FILE *url_fopen(const char *url, const char *operation) 233 { 234 /* this code could check for URLs or types in the 'url' and 235 basically use the real fopen() for standard files */ 236 237 URL_FILE *file; 238 (void)operation; 239 240 file = malloc(sizeof(URL_FILE)); 241 if(!file) 242 return NULL; 243 244 memset(file, 0, sizeof(URL_FILE)); 245 246 file->handle.file = fopen(url, operation); 247 if(file->handle.file) 248 file->type = CFTYPE_FILE; /* marked as URL */ 249 250 else { 251 file->type = CFTYPE_CURL; /* marked as URL */ 252 file->handle.curl = curl_easy_init(); 253 254 curl_easy_setopt(file->handle.curl, CURLOPT_URL, url); 255 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file); 256 curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L); 257 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback); 258 259 if(!multi_handle) 260 multi_handle = curl_multi_init(); 261 262 curl_multi_add_handle(multi_handle, file->handle.curl); 263 264 /* lets start the fetch */ 265 curl_multi_perform(multi_handle, &file->still_running); 266 267 if((file->buffer_pos == 0) && (!file->still_running)) { 268 /* if still_running is 0 now, we should return NULL */ 269 270 /* make sure the easy handle is not in the multi handle anymore */ 271 curl_multi_remove_handle(multi_handle, file->handle.curl); 272 273 /* cleanup */ 274 curl_easy_cleanup(file->handle.curl); 275 276 free(file); 277 278 file = NULL; 279 } 280 } 281 return file; 282 } 283 284 int url_fclose(URL_FILE *file) 285 { 286 int ret = 0;/* default is good return */ 287 288 switch(file->type) { 289 case CFTYPE_FILE: 290 ret = fclose(file->handle.file); /* passthrough */ 291 break; 292 293 case CFTYPE_CURL: 294 /* make sure the easy handle is not in the multi handle anymore */ 295 curl_multi_remove_handle(multi_handle, file->handle.curl); 296 297 /* cleanup */ 298 curl_easy_cleanup(file->handle.curl); 299 break; 300 301 default: /* unknown or supported type - oh dear */ 302 ret = EOF; 303 errno = EBADF; 304 break; 305 } 306 307 free(file->buffer);/* free any allocated buffer space */ 308 free(file); 309 310 return ret; 311 } 312 313 int url_feof(URL_FILE *file) 314 { 315 int ret = 0; 316 317 switch(file->type) { 318 case CFTYPE_FILE: 319 ret = feof(file->handle.file); 320 break; 321 322 case CFTYPE_CURL: 323 if((file->buffer_pos == 0) && (!file->still_running)) 324 ret = 1; 325 break; 326 327 default: /* unknown or supported type - oh dear */ 328 ret = -1; 329 errno = EBADF; 330 break; 331 } 332 return ret; 333 } 334 335 size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file) 336 { 337 size_t want; 338 339 switch(file->type) { 340 case CFTYPE_FILE: 341 want = fread(ptr, size, nmemb, file->handle.file); 342 break; 343 344 case CFTYPE_CURL: 345 want = nmemb * size; 346 347 fill_buffer(file, want); 348 349 /* check if there's data in the buffer - if not fill_buffer() 350 * either errored or EOF */ 351 if(!file->buffer_pos) 352 return 0; 353 354 /* ensure only available data is considered */ 355 if(file->buffer_pos < want) 356 want = file->buffer_pos; 357 358 /* xfer data to caller */ 359 memcpy(ptr, file->buffer, want); 360 361 use_buffer(file, want); 362 363 want = want / size; /* number of items */ 364 break; 365 366 default: /* unknown or supported type - oh dear */ 367 want = 0; 368 errno = EBADF; 369 break; 370 371 } 372 return want; 373 } 374 375 char *url_fgets(char *ptr, size_t size, URL_FILE *file) 376 { 377 size_t want = size - 1;/* always need to leave room for zero termination */ 378 size_t loop; 379 380 switch(file->type) { 381 case CFTYPE_FILE: 382 ptr = fgets(ptr, (int)size, file->handle.file); 383 break; 384 385 case CFTYPE_CURL: 386 fill_buffer(file, want); 387 388 /* check if there's data in the buffer - if not fill either errored or 389 * EOF */ 390 if(!file->buffer_pos) 391 return NULL; 392 393 /* ensure only available data is considered */ 394 if(file->buffer_pos < want) 395 want = file->buffer_pos; 396 397 /*buffer contains data */ 398 /* look for newline or eof */ 399 for(loop = 0; loop < want; loop++) { 400 if(file->buffer[loop] == '\n') { 401 want = loop + 1;/* include newline */ 402 break; 403 } 404 } 405 406 /* xfer data to caller */ 407 memcpy(ptr, file->buffer, want); 408 ptr[want] = 0;/* always null terminate */ 409 410 use_buffer(file, want); 411 412 break; 413 414 default: /* unknown or supported type - oh dear */ 415 ptr = NULL; 416 errno = EBADF; 417 break; 418 } 419 420 return ptr;/*success */ 421 } 422 423 void url_rewind(URL_FILE *file) 424 { 425 switch(file->type) { 426 case CFTYPE_FILE: 427 rewind(file->handle.file); /* passthrough */ 428 break; 429 430 case CFTYPE_CURL: 431 /* halt transaction */ 432 curl_multi_remove_handle(multi_handle, file->handle.curl); 433 434 /* restart */ 435 curl_multi_add_handle(multi_handle, file->handle.curl); 436 437 /* ditch buffer - write will recreate - resets stream pos*/ 438 free(file->buffer); 439 file->buffer = NULL; 440 file->buffer_pos = 0; 441 file->buffer_len = 0; 442 443 break; 444 445 default: /* unknown or supported type - oh dear */ 446 break; 447 } 448 } 449 450 #define FGETSFILE "fgets.test" 451 #define FREADFILE "fread.test" 452 #define REWINDFILE "rewind.test" 453 454 /* Small main program to retrieve from a url using fgets and fread saving the 455 * output to two test files (note the fgets method will corrupt binary files if 456 * they contain 0 chars */ 457 int main(int argc, char *argv[]) 458 { 459 URL_FILE *handle; 460 FILE *outf; 461 462 size_t nread; 463 char buffer[256]; 464 const char *url; 465 466 if(argc < 2) 467 url = "http://192.168.7.3/testfile";/* default to testurl */ 468 else 469 url = argv[1];/* use passed url */ 470 471 /* copy from url line by line with fgets */ 472 outf = fopen(FGETSFILE, "wb+"); 473 if(!outf) { 474 perror("couldn't open fgets output file\n"); 475 return 1; 476 } 477 478 handle = url_fopen(url, "r"); 479 if(!handle) { 480 printf("couldn't url_fopen() %s\n", url); 481 fclose(outf); 482 return 2; 483 } 484 485 while(!url_feof(handle)) { 486 url_fgets(buffer, sizeof(buffer), handle); 487 fwrite(buffer, 1, strlen(buffer), outf); 488 } 489 490 url_fclose(handle); 491 492 fclose(outf); 493 494 495 /* Copy from url with fread */ 496 outf = fopen(FREADFILE, "wb+"); 497 if(!outf) { 498 perror("couldn't open fread output file\n"); 499 return 1; 500 } 501 502 handle = url_fopen("testfile", "r"); 503 if(!handle) { 504 printf("couldn't url_fopen() testfile\n"); 505 fclose(outf); 506 return 2; 507 } 508 509 do { 510 nread = url_fread(buffer, 1, sizeof(buffer), handle); 511 fwrite(buffer, 1, nread, outf); 512 } while(nread); 513 514 url_fclose(handle); 515 516 fclose(outf); 517 518 519 /* Test rewind */ 520 outf = fopen(REWINDFILE, "wb+"); 521 if(!outf) { 522 perror("couldn't open fread output file\n"); 523 return 1; 524 } 525 526 handle = url_fopen("testfile", "r"); 527 if(!handle) { 528 printf("couldn't url_fopen() testfile\n"); 529 fclose(outf); 530 return 2; 531 } 532 533 nread = url_fread(buffer, 1, sizeof(buffer), handle); 534 fwrite(buffer, 1, nread, outf); 535 url_rewind(handle); 536 537 buffer[0]='\n'; 538 fwrite(buffer, 1, 1, outf); 539 540 nread = url_fread(buffer, 1, sizeof(buffer), handle); 541 fwrite(buffer, 1, nread, outf); 542 543 url_fclose(handle); 544 545 fclose(outf); 546 547 return 0;/* all done */ 548 } 549