1 /***************************************************************************** 2 * 3 * This example source code introduces a c library buffered I/O interface to 4 * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(), 5 * rewind(). Supported functions have identical prototypes to their normal c 6 * lib namesakes and are preceaded by url_ . 7 * 8 * Using this code you can replace your program's fopen() with url_fopen() 9 * and fread() with url_fread() and it become possible to read remote streams 10 * instead of (only) local files. Local files (ie those that can be directly 11 * fopened) will drop back to using the underlying clib implementations 12 * 13 * See the main() function at the bottom that shows an app that retrives from a 14 * specified url using fgets() and fread() and saves as two output files. 15 * 16 * Copyright (c) 2003 Simtec Electronics 17 * 18 * Re-implemented by Vincent Sanders <vince (at) kyllikki.org> with extensive 19 * reference to original curl example code 20 * 21 * Redistribution and use in source and binary forms, with or without 22 * modification, are permitted provided that the following conditions 23 * are met: 24 * 1. Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * 2. Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in the 28 * documentation and/or other materials provided with the distribution. 29 * 3. The name of the author may not be used to endorse or promote products 30 * derived from this software without specific prior written permission. 31 * 32 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 33 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 34 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 35 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 36 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 37 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 41 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 42 * 43 * This example requires libcurl 7.9.7 or later. 44 */ 45 46 #include <stdio.h> 47 #include <string.h> 48 #ifndef WIN32 49 # include <sys/time.h> 50 #endif 51 #include <stdlib.h> 52 #include <errno.h> 53 54 #include <curl/curl.h> 55 56 enum fcurl_type_e { 57 CFTYPE_NONE=0, 58 CFTYPE_FILE=1, 59 CFTYPE_CURL=2 60 }; 61 62 struct fcurl_data 63 { 64 enum fcurl_type_e type; /* type of handle */ 65 union { 66 CURL *curl; 67 FILE *file; 68 } handle; /* handle */ 69 70 char *buffer; /* buffer to store cached data*/ 71 size_t buffer_len; /* currently allocated buffers length */ 72 size_t buffer_pos; /* end of data in buffer*/ 73 int still_running; /* Is background url fetch still in progress */ 74 }; 75 76 typedef struct fcurl_data URL_FILE; 77 78 /* exported functions */ 79 URL_FILE *url_fopen(const char *url,const char *operation); 80 int url_fclose(URL_FILE *file); 81 int url_feof(URL_FILE *file); 82 size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file); 83 char * url_fgets(char *ptr, size_t size, URL_FILE *file); 84 void url_rewind(URL_FILE *file); 85 86 /* we use a global one for convenience */ 87 CURLM *multi_handle; 88 89 /* curl calls this routine to get more data */ 90 static size_t write_callback(char *buffer, 91 size_t size, 92 size_t nitems, 93 void *userp) 94 { 95 char *newbuff; 96 size_t rembuff; 97 98 URL_FILE *url = (URL_FILE *)userp; 99 size *= nitems; 100 101 rembuff=url->buffer_len - url->buffer_pos; /* remaining space in buffer */ 102 103 if(size > rembuff) { 104 /* not enough space in buffer */ 105 newbuff=realloc(url->buffer,url->buffer_len + (size - rembuff)); 106 if(newbuff==NULL) { 107 fprintf(stderr,"callback buffer grow failed\n"); 108 size=rembuff; 109 } 110 else { 111 /* realloc succeeded increase buffer size*/ 112 url->buffer_len+=size - rembuff; 113 url->buffer=newbuff; 114 } 115 } 116 117 memcpy(&url->buffer[url->buffer_pos], buffer, size); 118 url->buffer_pos += size; 119 120 return size; 121 } 122 123 /* use to attempt to fill the read buffer up to requested number of bytes */ 124 static int fill_buffer(URL_FILE *file, size_t want) 125 { 126 fd_set fdread; 127 fd_set fdwrite; 128 fd_set fdexcep; 129 struct timeval timeout; 130 int rc; 131 CURLMcode mc; /* curl_multi_fdset() return code */ 132 133 /* only attempt to fill buffer if transactions still running and buffer 134 * doesn't exceed required size already 135 */ 136 if((!file->still_running) || (file->buffer_pos > want)) 137 return 0; 138 139 /* attempt to fill buffer */ 140 do { 141 int maxfd = -1; 142 long curl_timeo = -1; 143 144 FD_ZERO(&fdread); 145 FD_ZERO(&fdwrite); 146 FD_ZERO(&fdexcep); 147 148 /* set a suitable timeout to fail on */ 149 timeout.tv_sec = 60; /* 1 minute */ 150 timeout.tv_usec = 0; 151 152 curl_multi_timeout(multi_handle, &curl_timeo); 153 if(curl_timeo >= 0) { 154 timeout.tv_sec = curl_timeo / 1000; 155 if(timeout.tv_sec > 1) 156 timeout.tv_sec = 1; 157 else 158 timeout.tv_usec = (curl_timeo % 1000) * 1000; 159 } 160 161 /* get file descriptors from the transfers */ 162 mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd); 163 164 if(mc != CURLM_OK) 165 { 166 fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc); 167 break; 168 } 169 170 /* On success the value of maxfd is guaranteed to be >= -1. We call 171 select(maxfd + 1, ...); specially in case of (maxfd == -1) there are 172 no fds ready yet so we call select(0, ...) --or Sleep() on Windows-- 173 to sleep 100ms, which is the minimum suggested value in the 174 curl_multi_fdset() doc. */ 175 176 if(maxfd == -1) { 177 #ifdef _WIN32 178 Sleep(100); 179 rc = 0; 180 #else 181 /* Portable sleep for platforms other than Windows. */ 182 struct timeval wait = { 0, 100 * 1000 }; /* 100ms */ 183 rc = select(0, NULL, NULL, NULL, &wait); 184 #endif 185 } 186 else { 187 /* Note that on some platforms 'timeout' may be modified by select(). 188 If you need access to the original value save a copy beforehand. */ 189 rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout); 190 } 191 192 switch(rc) { 193 case -1: 194 /* select error */ 195 break; 196 197 case 0: 198 default: 199 /* timeout or readable/writable sockets */ 200 curl_multi_perform(multi_handle, &file->still_running); 201 break; 202 } 203 } while(file->still_running && (file->buffer_pos < want)); 204 return 1; 205 } 206 207 /* use to remove want bytes from the front of a files buffer */ 208 static int use_buffer(URL_FILE *file, size_t want) 209 { 210 /* sort out buffer */ 211 if((file->buffer_pos - want) <=0) { 212 /* ditch buffer - write will recreate */ 213 free(file->buffer); 214 file->buffer=NULL; 215 file->buffer_pos=0; 216 file->buffer_len=0; 217 } 218 else { 219 /* move rest down make it available for later */ 220 memmove(file->buffer, 221 &file->buffer[want], 222 (file->buffer_pos - want)); 223 224 file->buffer_pos -= want; 225 } 226 return 0; 227 } 228 229 URL_FILE *url_fopen(const char *url,const char *operation) 230 { 231 /* this code could check for URLs or types in the 'url' and 232 basically use the real fopen() for standard files */ 233 234 URL_FILE *file; 235 (void)operation; 236 237 file = malloc(sizeof(URL_FILE)); 238 if(!file) 239 return NULL; 240 241 memset(file, 0, sizeof(URL_FILE)); 242 243 if((file->handle.file=fopen(url,operation))) 244 file->type = CFTYPE_FILE; /* marked as URL */ 245 246 else { 247 file->type = CFTYPE_CURL; /* marked as URL */ 248 file->handle.curl = curl_easy_init(); 249 250 curl_easy_setopt(file->handle.curl, CURLOPT_URL, url); 251 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file); 252 curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L); 253 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback); 254 255 if(!multi_handle) 256 multi_handle = curl_multi_init(); 257 258 curl_multi_add_handle(multi_handle, file->handle.curl); 259 260 /* lets start the fetch */ 261 curl_multi_perform(multi_handle, &file->still_running); 262 263 if((file->buffer_pos == 0) && (!file->still_running)) { 264 /* if still_running is 0 now, we should return NULL */ 265 266 /* make sure the easy handle is not in the multi handle anymore */ 267 curl_multi_remove_handle(multi_handle, file->handle.curl); 268 269 /* cleanup */ 270 curl_easy_cleanup(file->handle.curl); 271 272 free(file); 273 274 file = NULL; 275 } 276 } 277 return file; 278 } 279 280 int url_fclose(URL_FILE *file) 281 { 282 int ret=0;/* default is good return */ 283 284 switch(file->type) { 285 case CFTYPE_FILE: 286 ret=fclose(file->handle.file); /* passthrough */ 287 break; 288 289 case CFTYPE_CURL: 290 /* make sure the easy handle is not in the multi handle anymore */ 291 curl_multi_remove_handle(multi_handle, file->handle.curl); 292 293 /* cleanup */ 294 curl_easy_cleanup(file->handle.curl); 295 break; 296 297 default: /* unknown or supported type - oh dear */ 298 ret=EOF; 299 errno=EBADF; 300 break; 301 } 302 303 free(file->buffer);/* free any allocated buffer space */ 304 free(file); 305 306 return ret; 307 } 308 309 int url_feof(URL_FILE *file) 310 { 311 int ret=0; 312 313 switch(file->type) { 314 case CFTYPE_FILE: 315 ret=feof(file->handle.file); 316 break; 317 318 case CFTYPE_CURL: 319 if((file->buffer_pos == 0) && (!file->still_running)) 320 ret = 1; 321 break; 322 323 default: /* unknown or supported type - oh dear */ 324 ret=-1; 325 errno=EBADF; 326 break; 327 } 328 return ret; 329 } 330 331 size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file) 332 { 333 size_t want; 334 335 switch(file->type) { 336 case CFTYPE_FILE: 337 want=fread(ptr,size,nmemb,file->handle.file); 338 break; 339 340 case CFTYPE_CURL: 341 want = nmemb * size; 342 343 fill_buffer(file,want); 344 345 /* check if theres data in the buffer - if not fill_buffer() 346 * either errored or EOF */ 347 if(!file->buffer_pos) 348 return 0; 349 350 /* ensure only available data is considered */ 351 if(file->buffer_pos < want) 352 want = file->buffer_pos; 353 354 /* xfer data to caller */ 355 memcpy(ptr, file->buffer, want); 356 357 use_buffer(file,want); 358 359 want = want / size; /* number of items */ 360 break; 361 362 default: /* unknown or supported type - oh dear */ 363 want=0; 364 errno=EBADF; 365 break; 366 367 } 368 return want; 369 } 370 371 char *url_fgets(char *ptr, size_t size, URL_FILE *file) 372 { 373 size_t want = size - 1;/* always need to leave room for zero termination */ 374 size_t loop; 375 376 switch(file->type) { 377 case CFTYPE_FILE: 378 ptr = fgets(ptr, (int)size, file->handle.file); 379 break; 380 381 case CFTYPE_CURL: 382 fill_buffer(file,want); 383 384 /* check if theres data in the buffer - if not fill either errored or 385 * EOF */ 386 if(!file->buffer_pos) 387 return NULL; 388 389 /* ensure only available data is considered */ 390 if(file->buffer_pos < want) 391 want = file->buffer_pos; 392 393 /*buffer contains data */ 394 /* look for newline or eof */ 395 for(loop=0;loop < want;loop++) { 396 if(file->buffer[loop] == '\n') { 397 want=loop+1;/* include newline */ 398 break; 399 } 400 } 401 402 /* xfer data to caller */ 403 memcpy(ptr, file->buffer, want); 404 ptr[want]=0;/* allways null terminate */ 405 406 use_buffer(file,want); 407 408 break; 409 410 default: /* unknown or supported type - oh dear */ 411 ptr=NULL; 412 errno=EBADF; 413 break; 414 } 415 416 return ptr;/*success */ 417 } 418 419 void url_rewind(URL_FILE *file) 420 { 421 switch(file->type) { 422 case CFTYPE_FILE: 423 rewind(file->handle.file); /* passthrough */ 424 break; 425 426 case CFTYPE_CURL: 427 /* halt transaction */ 428 curl_multi_remove_handle(multi_handle, file->handle.curl); 429 430 /* restart */ 431 curl_multi_add_handle(multi_handle, file->handle.curl); 432 433 /* ditch buffer - write will recreate - resets stream pos*/ 434 free(file->buffer); 435 file->buffer=NULL; 436 file->buffer_pos=0; 437 file->buffer_len=0; 438 439 break; 440 441 default: /* unknown or supported type - oh dear */ 442 break; 443 } 444 } 445 446 /* Small main program to retrive from a url using fgets and fread saving the 447 * output to two test files (note the fgets method will corrupt binary files if 448 * they contain 0 chars */ 449 int main(int argc, char *argv[]) 450 { 451 URL_FILE *handle; 452 FILE *outf; 453 454 size_t nread; 455 char buffer[256]; 456 const char *url; 457 458 if(argc < 2) 459 url="http://192.168.7.3/testfile";/* default to testurl */ 460 else 461 url=argv[1];/* use passed url */ 462 463 /* copy from url line by line with fgets */ 464 outf=fopen("fgets.test","w+"); 465 if(!outf) { 466 perror("couldn't open fgets output file\n"); 467 return 1; 468 } 469 470 handle = url_fopen(url, "r"); 471 if(!handle) { 472 printf("couldn't url_fopen() %s\n", url); 473 fclose(outf); 474 return 2; 475 } 476 477 while(!url_feof(handle)) { 478 url_fgets(buffer,sizeof(buffer),handle); 479 fwrite(buffer,1,strlen(buffer),outf); 480 } 481 482 url_fclose(handle); 483 484 fclose(outf); 485 486 487 /* Copy from url with fread */ 488 outf=fopen("fread.test","w+"); 489 if(!outf) { 490 perror("couldn't open fread output file\n"); 491 return 1; 492 } 493 494 handle = url_fopen("testfile", "r"); 495 if(!handle) { 496 printf("couldn't url_fopen() testfile\n"); 497 fclose(outf); 498 return 2; 499 } 500 501 do { 502 nread = url_fread(buffer, 1, sizeof(buffer), handle); 503 fwrite(buffer,1,nread,outf); 504 } while(nread); 505 506 url_fclose(handle); 507 508 fclose(outf); 509 510 511 /* Test rewind */ 512 outf=fopen("rewind.test","w+"); 513 if(!outf) { 514 perror("couldn't open fread output file\n"); 515 return 1; 516 } 517 518 handle = url_fopen("testfile", "r"); 519 if(!handle) { 520 printf("couldn't url_fopen() testfile\n"); 521 fclose(outf); 522 return 2; 523 } 524 525 nread = url_fread(buffer, 1,sizeof(buffer), handle); 526 fwrite(buffer,1,nread,outf); 527 url_rewind(handle); 528 529 buffer[0]='\n'; 530 fwrite(buffer,1,1,outf); 531 532 nread = url_fread(buffer, 1,sizeof(buffer), handle); 533 fwrite(buffer,1,nread,outf); 534 535 536 url_fclose(handle); 537 538 fclose(outf); 539 540 541 return 0;/* all done */ 542 } 543