1 /***************************************************************************** 2 * 3 * This example source code introduces a c library buffered I/O interface to 4 * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(), 5 * rewind(). Supported functions have identical prototypes to their normal c 6 * lib namesakes and are preceaded by url_ . 7 * 8 * Using this code you can replace your program's fopen() with url_fopen() 9 * and fread() with url_fread() and it become possible to read remote streams 10 * instead of (only) local files. Local files (ie those that can be directly 11 * fopened) will drop back to using the underlying clib implementations 12 * 13 * See the main() function at the bottom that shows an app that retrives from a 14 * specified url using fgets() and fread() and saves as two output files. 15 * 16 * Copyright (c) 2003 Simtec Electronics 17 * 18 * Re-implemented by Vincent Sanders <vince (at) kyllikki.org> with extensive 19 * reference to original curl example code 20 * 21 * Redistribution and use in source and binary forms, with or without 22 * modification, are permitted provided that the following conditions 23 * are met: 24 * 1. Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * 2. Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in the 28 * documentation and/or other materials provided with the distribution. 29 * 3. The name of the author may not be used to endorse or promote products 30 * derived from this software without specific prior written permission. 31 * 32 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 33 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 34 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 35 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 36 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 37 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 41 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 42 * 43 * This example requires libcurl 7.9.7 or later. 44 */ 45 /* <DESC> 46 * implements an fopen() abstraction allowing reading from URLs 47 * </DESC> 48 */ 49 50 #include <stdio.h> 51 #include <string.h> 52 #ifndef WIN32 53 # include <sys/time.h> 54 #endif 55 #include <stdlib.h> 56 #include <errno.h> 57 58 #include <curl/curl.h> 59 60 enum fcurl_type_e { 61 CFTYPE_NONE=0, 62 CFTYPE_FILE=1, 63 CFTYPE_CURL=2 64 }; 65 66 struct fcurl_data 67 { 68 enum fcurl_type_e type; /* type of handle */ 69 union { 70 CURL *curl; 71 FILE *file; 72 } handle; /* handle */ 73 74 char *buffer; /* buffer to store cached data*/ 75 size_t buffer_len; /* currently allocated buffers length */ 76 size_t buffer_pos; /* end of data in buffer*/ 77 int still_running; /* Is background url fetch still in progress */ 78 }; 79 80 typedef struct fcurl_data URL_FILE; 81 82 /* exported functions */ 83 URL_FILE *url_fopen(const char *url, const char *operation); 84 int url_fclose(URL_FILE *file); 85 int url_feof(URL_FILE *file); 86 size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file); 87 char * url_fgets(char *ptr, size_t size, URL_FILE *file); 88 void url_rewind(URL_FILE *file); 89 90 /* we use a global one for convenience */ 91 CURLM *multi_handle; 92 93 /* curl calls this routine to get more data */ 94 static size_t write_callback(char *buffer, 95 size_t size, 96 size_t nitems, 97 void *userp) 98 { 99 char *newbuff; 100 size_t rembuff; 101 102 URL_FILE *url = (URL_FILE *)userp; 103 size *= nitems; 104 105 rembuff=url->buffer_len - url->buffer_pos; /* remaining space in buffer */ 106 107 if(size > rembuff) { 108 /* not enough space in buffer */ 109 newbuff=realloc(url->buffer, url->buffer_len + (size - rembuff)); 110 if(newbuff==NULL) { 111 fprintf(stderr, "callback buffer grow failed\n"); 112 size=rembuff; 113 } 114 else { 115 /* realloc succeeded increase buffer size*/ 116 url->buffer_len+=size - rembuff; 117 url->buffer=newbuff; 118 } 119 } 120 121 memcpy(&url->buffer[url->buffer_pos], buffer, size); 122 url->buffer_pos += size; 123 124 return size; 125 } 126 127 /* use to attempt to fill the read buffer up to requested number of bytes */ 128 static int fill_buffer(URL_FILE *file, size_t want) 129 { 130 fd_set fdread; 131 fd_set fdwrite; 132 fd_set fdexcep; 133 struct timeval timeout; 134 int rc; 135 CURLMcode mc; /* curl_multi_fdset() return code */ 136 137 /* only attempt to fill buffer if transactions still running and buffer 138 * doesn't exceed required size already 139 */ 140 if((!file->still_running) || (file->buffer_pos > want)) 141 return 0; 142 143 /* attempt to fill buffer */ 144 do { 145 int maxfd = -1; 146 long curl_timeo = -1; 147 148 FD_ZERO(&fdread); 149 FD_ZERO(&fdwrite); 150 FD_ZERO(&fdexcep); 151 152 /* set a suitable timeout to fail on */ 153 timeout.tv_sec = 60; /* 1 minute */ 154 timeout.tv_usec = 0; 155 156 curl_multi_timeout(multi_handle, &curl_timeo); 157 if(curl_timeo >= 0) { 158 timeout.tv_sec = curl_timeo / 1000; 159 if(timeout.tv_sec > 1) 160 timeout.tv_sec = 1; 161 else 162 timeout.tv_usec = (curl_timeo % 1000) * 1000; 163 } 164 165 /* get file descriptors from the transfers */ 166 mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd); 167 168 if(mc != CURLM_OK) { 169 fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc); 170 break; 171 } 172 173 /* On success the value of maxfd is guaranteed to be >= -1. We call 174 select(maxfd + 1, ...); specially in case of (maxfd == -1) there are 175 no fds ready yet so we call select(0, ...) --or Sleep() on Windows-- 176 to sleep 100ms, which is the minimum suggested value in the 177 curl_multi_fdset() doc. */ 178 179 if(maxfd == -1) { 180 #ifdef _WIN32 181 Sleep(100); 182 rc = 0; 183 #else 184 /* Portable sleep for platforms other than Windows. */ 185 struct timeval wait = { 0, 100 * 1000 }; /* 100ms */ 186 rc = select(0, NULL, NULL, NULL, &wait); 187 #endif 188 } 189 else { 190 /* Note that on some platforms 'timeout' may be modified by select(). 191 If you need access to the original value save a copy beforehand. */ 192 rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout); 193 } 194 195 switch(rc) { 196 case -1: 197 /* select error */ 198 break; 199 200 case 0: 201 default: 202 /* timeout or readable/writable sockets */ 203 curl_multi_perform(multi_handle, &file->still_running); 204 break; 205 } 206 } while(file->still_running && (file->buffer_pos < want)); 207 return 1; 208 } 209 210 /* use to remove want bytes from the front of a files buffer */ 211 static int use_buffer(URL_FILE *file, size_t want) 212 { 213 /* sort out buffer */ 214 if((file->buffer_pos - want) <=0) { 215 /* ditch buffer - write will recreate */ 216 free(file->buffer); 217 file->buffer=NULL; 218 file->buffer_pos=0; 219 file->buffer_len=0; 220 } 221 else { 222 /* move rest down make it available for later */ 223 memmove(file->buffer, 224 &file->buffer[want], 225 (file->buffer_pos - want)); 226 227 file->buffer_pos -= want; 228 } 229 return 0; 230 } 231 232 URL_FILE *url_fopen(const char *url, const char *operation) 233 { 234 /* this code could check for URLs or types in the 'url' and 235 basically use the real fopen() for standard files */ 236 237 URL_FILE *file; 238 (void)operation; 239 240 file = malloc(sizeof(URL_FILE)); 241 if(!file) 242 return NULL; 243 244 memset(file, 0, sizeof(URL_FILE)); 245 246 if((file->handle.file=fopen(url, operation))) 247 file->type = CFTYPE_FILE; /* marked as URL */ 248 249 else { 250 file->type = CFTYPE_CURL; /* marked as URL */ 251 file->handle.curl = curl_easy_init(); 252 253 curl_easy_setopt(file->handle.curl, CURLOPT_URL, url); 254 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file); 255 curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L); 256 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback); 257 258 if(!multi_handle) 259 multi_handle = curl_multi_init(); 260 261 curl_multi_add_handle(multi_handle, file->handle.curl); 262 263 /* lets start the fetch */ 264 curl_multi_perform(multi_handle, &file->still_running); 265 266 if((file->buffer_pos == 0) && (!file->still_running)) { 267 /* if still_running is 0 now, we should return NULL */ 268 269 /* make sure the easy handle is not in the multi handle anymore */ 270 curl_multi_remove_handle(multi_handle, file->handle.curl); 271 272 /* cleanup */ 273 curl_easy_cleanup(file->handle.curl); 274 275 free(file); 276 277 file = NULL; 278 } 279 } 280 return file; 281 } 282 283 int url_fclose(URL_FILE *file) 284 { 285 int ret=0;/* default is good return */ 286 287 switch(file->type) { 288 case CFTYPE_FILE: 289 ret=fclose(file->handle.file); /* passthrough */ 290 break; 291 292 case CFTYPE_CURL: 293 /* make sure the easy handle is not in the multi handle anymore */ 294 curl_multi_remove_handle(multi_handle, file->handle.curl); 295 296 /* cleanup */ 297 curl_easy_cleanup(file->handle.curl); 298 break; 299 300 default: /* unknown or supported type - oh dear */ 301 ret=EOF; 302 errno=EBADF; 303 break; 304 } 305 306 free(file->buffer);/* free any allocated buffer space */ 307 free(file); 308 309 return ret; 310 } 311 312 int url_feof(URL_FILE *file) 313 { 314 int ret=0; 315 316 switch(file->type) { 317 case CFTYPE_FILE: 318 ret=feof(file->handle.file); 319 break; 320 321 case CFTYPE_CURL: 322 if((file->buffer_pos == 0) && (!file->still_running)) 323 ret = 1; 324 break; 325 326 default: /* unknown or supported type - oh dear */ 327 ret=-1; 328 errno=EBADF; 329 break; 330 } 331 return ret; 332 } 333 334 size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file) 335 { 336 size_t want; 337 338 switch(file->type) { 339 case CFTYPE_FILE: 340 want=fread(ptr, size, nmemb, file->handle.file); 341 break; 342 343 case CFTYPE_CURL: 344 want = nmemb * size; 345 346 fill_buffer(file, want); 347 348 /* check if theres data in the buffer - if not fill_buffer() 349 * either errored or EOF */ 350 if(!file->buffer_pos) 351 return 0; 352 353 /* ensure only available data is considered */ 354 if(file->buffer_pos < want) 355 want = file->buffer_pos; 356 357 /* xfer data to caller */ 358 memcpy(ptr, file->buffer, want); 359 360 use_buffer(file, want); 361 362 want = want / size; /* number of items */ 363 break; 364 365 default: /* unknown or supported type - oh dear */ 366 want=0; 367 errno=EBADF; 368 break; 369 370 } 371 return want; 372 } 373 374 char *url_fgets(char *ptr, size_t size, URL_FILE *file) 375 { 376 size_t want = size - 1;/* always need to leave room for zero termination */ 377 size_t loop; 378 379 switch(file->type) { 380 case CFTYPE_FILE: 381 ptr = fgets(ptr, (int)size, file->handle.file); 382 break; 383 384 case CFTYPE_CURL: 385 fill_buffer(file, want); 386 387 /* check if theres data in the buffer - if not fill either errored or 388 * EOF */ 389 if(!file->buffer_pos) 390 return NULL; 391 392 /* ensure only available data is considered */ 393 if(file->buffer_pos < want) 394 want = file->buffer_pos; 395 396 /*buffer contains data */ 397 /* look for newline or eof */ 398 for(loop=0;loop < want;loop++) { 399 if(file->buffer[loop] == '\n') { 400 want=loop+1;/* include newline */ 401 break; 402 } 403 } 404 405 /* xfer data to caller */ 406 memcpy(ptr, file->buffer, want); 407 ptr[want]=0;/* allways null terminate */ 408 409 use_buffer(file, want); 410 411 break; 412 413 default: /* unknown or supported type - oh dear */ 414 ptr=NULL; 415 errno=EBADF; 416 break; 417 } 418 419 return ptr;/*success */ 420 } 421 422 void url_rewind(URL_FILE *file) 423 { 424 switch(file->type) { 425 case CFTYPE_FILE: 426 rewind(file->handle.file); /* passthrough */ 427 break; 428 429 case CFTYPE_CURL: 430 /* halt transaction */ 431 curl_multi_remove_handle(multi_handle, file->handle.curl); 432 433 /* restart */ 434 curl_multi_add_handle(multi_handle, file->handle.curl); 435 436 /* ditch buffer - write will recreate - resets stream pos*/ 437 free(file->buffer); 438 file->buffer=NULL; 439 file->buffer_pos=0; 440 file->buffer_len=0; 441 442 break; 443 444 default: /* unknown or supported type - oh dear */ 445 break; 446 } 447 } 448 449 #define FGETSFILE "fgets.test" 450 #define FREADFILE "fread.test" 451 #define REWINDFILE "rewind.test" 452 453 /* Small main program to retrive from a url using fgets and fread saving the 454 * output to two test files (note the fgets method will corrupt binary files if 455 * they contain 0 chars */ 456 int main(int argc, char *argv[]) 457 { 458 URL_FILE *handle; 459 FILE *outf; 460 461 size_t nread; 462 char buffer[256]; 463 const char *url; 464 465 if(argc < 2) 466 url="http://192.168.7.3/testfile";/* default to testurl */ 467 else 468 url=argv[1];/* use passed url */ 469 470 /* copy from url line by line with fgets */ 471 outf=fopen(FGETSFILE, "wb+"); 472 if(!outf) { 473 perror("couldn't open fgets output file\n"); 474 return 1; 475 } 476 477 handle = url_fopen(url, "r"); 478 if(!handle) { 479 printf("couldn't url_fopen() %s\n", url); 480 fclose(outf); 481 return 2; 482 } 483 484 while(!url_feof(handle)) { 485 url_fgets(buffer, sizeof(buffer), handle); 486 fwrite(buffer, 1, strlen(buffer), outf); 487 } 488 489 url_fclose(handle); 490 491 fclose(outf); 492 493 494 /* Copy from url with fread */ 495 outf=fopen(FREADFILE, "wb+"); 496 if(!outf) { 497 perror("couldn't open fread output file\n"); 498 return 1; 499 } 500 501 handle = url_fopen("testfile", "r"); 502 if(!handle) { 503 printf("couldn't url_fopen() testfile\n"); 504 fclose(outf); 505 return 2; 506 } 507 508 do { 509 nread = url_fread(buffer, 1, sizeof(buffer), handle); 510 fwrite(buffer, 1, nread, outf); 511 } while(nread); 512 513 url_fclose(handle); 514 515 fclose(outf); 516 517 518 /* Test rewind */ 519 outf=fopen(REWINDFILE, "wb+"); 520 if(!outf) { 521 perror("couldn't open fread output file\n"); 522 return 1; 523 } 524 525 handle = url_fopen("testfile", "r"); 526 if(!handle) { 527 printf("couldn't url_fopen() testfile\n"); 528 fclose(outf); 529 return 2; 530 } 531 532 nread = url_fread(buffer, 1, sizeof(buffer), handle); 533 fwrite(buffer, 1, nread, outf); 534 url_rewind(handle); 535 536 buffer[0]='\n'; 537 fwrite(buffer, 1, 1, outf); 538 539 nread = url_fread(buffer, 1, sizeof(buffer), handle); 540 fwrite(buffer, 1, nread, outf); 541 542 url_fclose(handle); 543 544 fclose(outf); 545 546 return 0;/* all done */ 547 } 548