1 /*************************************************************************** 2 * _ _ ____ _ 3 * Project ___| | | | _ \| | 4 * / __| | | | |_) | | 5 * | (__| |_| | _ <| |___ 6 * \___|\___/|_| \_\_____| 7 * 8 * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel (at) haxx.se>, et al. 9 * 10 * This software is licensed as described in the file COPYING, which 11 * you should have received as part of this distribution. The terms 12 * are also available at https://curl.haxx.se/docs/copyright.html. 13 * 14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell 15 * copies of the Software, and permit persons to whom the Software is 16 * furnished to do so, under the terms of the COPYING file. 17 * 18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 19 * KIND, either express or implied. 20 * 21 ***************************************************************************/ 22 /* <DESC> 23 * multi socket API usage with libevent 2 24 * </DESC> 25 */ 26 /* Example application source code using the multi socket interface to 27 download many files at once. 28 29 Written by Jeff Pohlmeyer 30 31 Requires libevent version 2 and a (POSIX?) system that has mkfifo(). 32 33 This is an adaptation of libcurl's "hipev.c" and libevent's "event-test.c" 34 sample programs. 35 36 When running, the program creates the named pipe "hiper.fifo" 37 38 Whenever there is input into the fifo, the program reads the input as a list 39 of URL's and creates some new easy handles to fetch each URL via the 40 curl_multi "hiper" API. 41 42 43 Thus, you can try a single URL: 44 % echo http://www.yahoo.com > hiper.fifo 45 46 Or a whole bunch of them: 47 % cat my-url-list > hiper.fifo 48 49 The fifo buffer is handled almost instantly, so you can even add more URL's 50 while the previous requests are still being downloaded. 51 52 Note: 53 For the sake of simplicity, URL length is limited to 1023 char's ! 54 55 This is purely a demo app, all retrieved data is simply discarded by the write 56 callback. 57 58 */ 59 60 #include <stdio.h> 61 #include <string.h> 62 #include <stdlib.h> 63 #include <sys/time.h> 64 #include <time.h> 65 #include <unistd.h> 66 #include <sys/poll.h> 67 #include <curl/curl.h> 68 #include <event2/event.h> 69 #include <fcntl.h> 70 #include <sys/stat.h> 71 #include <errno.h> 72 73 74 #define MSG_OUT stdout /* Send info to stdout, change to stderr if you want */ 75 76 77 /* Global information, common to all connections */ 78 typedef struct _GlobalInfo 79 { 80 struct event_base *evbase; 81 struct event *fifo_event; 82 struct event *timer_event; 83 CURLM *multi; 84 int still_running; 85 FILE *input; 86 } GlobalInfo; 87 88 89 /* Information associated with a specific easy handle */ 90 typedef struct _ConnInfo 91 { 92 CURL *easy; 93 char *url; 94 GlobalInfo *global; 95 char error[CURL_ERROR_SIZE]; 96 } ConnInfo; 97 98 99 /* Information associated with a specific socket */ 100 typedef struct _SockInfo 101 { 102 curl_socket_t sockfd; 103 CURL *easy; 104 int action; 105 long timeout; 106 struct event *ev; 107 int evset; 108 GlobalInfo *global; 109 } SockInfo; 110 111 112 113 /* Update the event timer after curl_multi library calls */ 114 static int multi_timer_cb(CURLM *multi, long timeout_ms, GlobalInfo *g) 115 { 116 struct timeval timeout; 117 (void)multi; /* unused */ 118 119 timeout.tv_sec = timeout_ms/1000; 120 timeout.tv_usec = (timeout_ms%1000)*1000; 121 fprintf(MSG_OUT, "multi_timer_cb: Setting timeout to %ld ms\n", timeout_ms); 122 evtimer_add(g->timer_event, &timeout); 123 return 0; 124 } 125 126 /* Die if we get a bad CURLMcode somewhere */ 127 static void mcode_or_die(const char *where, CURLMcode code) 128 { 129 if(CURLM_OK != code) { 130 const char *s; 131 switch (code) { 132 case CURLM_BAD_HANDLE: s="CURLM_BAD_HANDLE"; break; 133 case CURLM_BAD_EASY_HANDLE: s="CURLM_BAD_EASY_HANDLE"; break; 134 case CURLM_OUT_OF_MEMORY: s="CURLM_OUT_OF_MEMORY"; break; 135 case CURLM_INTERNAL_ERROR: s="CURLM_INTERNAL_ERROR"; break; 136 case CURLM_UNKNOWN_OPTION: s="CURLM_UNKNOWN_OPTION"; break; 137 case CURLM_LAST: s="CURLM_LAST"; break; 138 default: s="CURLM_unknown"; 139 break; 140 case CURLM_BAD_SOCKET: s="CURLM_BAD_SOCKET"; 141 fprintf(MSG_OUT, "ERROR: %s returns %s\n", where, s); 142 /* ignore this error */ 143 return; 144 } 145 fprintf(MSG_OUT, "ERROR: %s returns %s\n", where, s); 146 exit(code); 147 } 148 } 149 150 151 152 /* Check for completed transfers, and remove their easy handles */ 153 static void check_multi_info(GlobalInfo *g) 154 { 155 char *eff_url; 156 CURLMsg *msg; 157 int msgs_left; 158 ConnInfo *conn; 159 CURL *easy; 160 CURLcode res; 161 162 fprintf(MSG_OUT, "REMAINING: %d\n", g->still_running); 163 while((msg = curl_multi_info_read(g->multi, &msgs_left))) { 164 if(msg->msg == CURLMSG_DONE) { 165 easy = msg->easy_handle; 166 res = msg->data.result; 167 curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn); 168 curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &eff_url); 169 fprintf(MSG_OUT, "DONE: %s => (%d) %s\n", eff_url, res, conn->error); 170 curl_multi_remove_handle(g->multi, easy); 171 free(conn->url); 172 curl_easy_cleanup(easy); 173 free(conn); 174 } 175 } 176 } 177 178 179 180 /* Called by libevent when we get action on a multi socket */ 181 static void event_cb(int fd, short kind, void *userp) 182 { 183 GlobalInfo *g = (GlobalInfo*) userp; 184 CURLMcode rc; 185 186 int action = 187 (kind & EV_READ ? CURL_CSELECT_IN : 0) | 188 (kind & EV_WRITE ? CURL_CSELECT_OUT : 0); 189 190 rc = curl_multi_socket_action(g->multi, fd, action, &g->still_running); 191 mcode_or_die("event_cb: curl_multi_socket_action", rc); 192 193 check_multi_info(g); 194 if(g->still_running <= 0) { 195 fprintf(MSG_OUT, "last transfer done, kill timeout\n"); 196 if(evtimer_pending(g->timer_event, NULL)) { 197 evtimer_del(g->timer_event); 198 } 199 } 200 } 201 202 203 204 /* Called by libevent when our timeout expires */ 205 static void timer_cb(int fd, short kind, void *userp) 206 { 207 GlobalInfo *g = (GlobalInfo *)userp; 208 CURLMcode rc; 209 (void)fd; 210 (void)kind; 211 212 rc = curl_multi_socket_action(g->multi, 213 CURL_SOCKET_TIMEOUT, 0, &g->still_running); 214 mcode_or_die("timer_cb: curl_multi_socket_action", rc); 215 check_multi_info(g); 216 } 217 218 219 220 /* Clean up the SockInfo structure */ 221 static void remsock(SockInfo *f) 222 { 223 if(f) { 224 if(f->evset) 225 event_free(f->ev); 226 free(f); 227 } 228 } 229 230 231 232 /* Assign information to a SockInfo structure */ 233 static void setsock(SockInfo *f, curl_socket_t s, CURL *e, int act, 234 GlobalInfo *g) 235 { 236 int kind = 237 (act&CURL_POLL_IN?EV_READ:0)|(act&CURL_POLL_OUT?EV_WRITE:0)|EV_PERSIST; 238 239 f->sockfd = s; 240 f->action = act; 241 f->easy = e; 242 if(f->evset) 243 event_free(f->ev); 244 f->ev = event_new(g->evbase, f->sockfd, kind, event_cb, g); 245 f->evset = 1; 246 event_add(f->ev, NULL); 247 } 248 249 250 251 /* Initialize a new SockInfo structure */ 252 static void addsock(curl_socket_t s, CURL *easy, int action, GlobalInfo *g) 253 { 254 SockInfo *fdp = calloc(sizeof(SockInfo), 1); 255 256 fdp->global = g; 257 setsock(fdp, s, easy, action, g); 258 curl_multi_assign(g->multi, s, fdp); 259 } 260 261 /* CURLMOPT_SOCKETFUNCTION */ 262 static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp) 263 { 264 GlobalInfo *g = (GlobalInfo*) cbp; 265 SockInfo *fdp = (SockInfo*) sockp; 266 const char *whatstr[]={ "none", "IN", "OUT", "INOUT", "REMOVE" }; 267 268 fprintf(MSG_OUT, 269 "socket callback: s=%d e=%p what=%s ", s, e, whatstr[what]); 270 if(what == CURL_POLL_REMOVE) { 271 fprintf(MSG_OUT, "\n"); 272 remsock(fdp); 273 } 274 else { 275 if(!fdp) { 276 fprintf(MSG_OUT, "Adding data: %s\n", whatstr[what]); 277 addsock(s, e, what, g); 278 } 279 else { 280 fprintf(MSG_OUT, 281 "Changing action from %s to %s\n", 282 whatstr[fdp->action], whatstr[what]); 283 setsock(fdp, s, e, what, g); 284 } 285 } 286 return 0; 287 } 288 289 290 291 /* CURLOPT_WRITEFUNCTION */ 292 static size_t write_cb(void *ptr, size_t size, size_t nmemb, void *data) 293 { 294 size_t realsize = size * nmemb; 295 ConnInfo *conn = (ConnInfo*) data; 296 (void)ptr; 297 (void)conn; 298 return realsize; 299 } 300 301 302 /* CURLOPT_PROGRESSFUNCTION */ 303 static int prog_cb (void *p, double dltotal, double dlnow, double ult, 304 double uln) 305 { 306 ConnInfo *conn = (ConnInfo *)p; 307 (void)ult; 308 (void)uln; 309 310 fprintf(MSG_OUT, "Progress: %s (%g/%g)\n", conn->url, dlnow, dltotal); 311 return 0; 312 } 313 314 315 /* Create a new easy handle, and add it to the global curl_multi */ 316 static void new_conn(char *url, GlobalInfo *g) 317 { 318 ConnInfo *conn; 319 CURLMcode rc; 320 321 conn = calloc(1, sizeof(ConnInfo)); 322 memset(conn, 0, sizeof(ConnInfo)); 323 conn->error[0]='\0'; 324 325 conn->easy = curl_easy_init(); 326 if(!conn->easy) { 327 fprintf(MSG_OUT, "curl_easy_init() failed, exiting!\n"); 328 exit(2); 329 } 330 conn->global = g; 331 conn->url = strdup(url); 332 curl_easy_setopt(conn->easy, CURLOPT_URL, conn->url); 333 curl_easy_setopt(conn->easy, CURLOPT_WRITEFUNCTION, write_cb); 334 curl_easy_setopt(conn->easy, CURLOPT_WRITEDATA, conn); 335 curl_easy_setopt(conn->easy, CURLOPT_VERBOSE, 1L); 336 curl_easy_setopt(conn->easy, CURLOPT_ERRORBUFFER, conn->error); 337 curl_easy_setopt(conn->easy, CURLOPT_PRIVATE, conn); 338 curl_easy_setopt(conn->easy, CURLOPT_NOPROGRESS, 0L); 339 curl_easy_setopt(conn->easy, CURLOPT_PROGRESSFUNCTION, prog_cb); 340 curl_easy_setopt(conn->easy, CURLOPT_PROGRESSDATA, conn); 341 fprintf(MSG_OUT, 342 "Adding easy %p to multi %p (%s)\n", conn->easy, g->multi, url); 343 rc = curl_multi_add_handle(g->multi, conn->easy); 344 mcode_or_die("new_conn: curl_multi_add_handle", rc); 345 346 /* note that the add_handle() will set a time-out to trigger very soon so 347 that the necessary socket_action() call will be called by this app */ 348 } 349 350 /* This gets called whenever data is received from the fifo */ 351 static void fifo_cb(int fd, short event, void *arg) 352 { 353 char s[1024]; 354 long int rv=0; 355 int n=0; 356 GlobalInfo *g = (GlobalInfo *)arg; 357 (void)fd; /* unused */ 358 (void)event; /* unused */ 359 360 do { 361 s[0]='\0'; 362 rv=fscanf(g->input, "%1023s%n", s, &n); 363 s[n]='\0'; 364 if(n && s[0]) { 365 new_conn(s, arg); /* if we read a URL, go get it! */ 366 } 367 else 368 break; 369 } while(rv != EOF); 370 } 371 372 /* Create a named pipe and tell libevent to monitor it */ 373 static const char *fifo = "hiper.fifo"; 374 static int init_fifo (GlobalInfo *g) 375 { 376 struct stat st; 377 curl_socket_t sockfd; 378 379 fprintf(MSG_OUT, "Creating named pipe \"%s\"\n", fifo); 380 if(lstat (fifo, &st) == 0) { 381 if((st.st_mode & S_IFMT) == S_IFREG) { 382 errno = EEXIST; 383 perror("lstat"); 384 exit (1); 385 } 386 } 387 unlink(fifo); 388 if(mkfifo (fifo, 0600) == -1) { 389 perror("mkfifo"); 390 exit (1); 391 } 392 sockfd = open(fifo, O_RDWR | O_NONBLOCK, 0); 393 if(sockfd == -1) { 394 perror("open"); 395 exit (1); 396 } 397 g->input = fdopen(sockfd, "r"); 398 399 fprintf(MSG_OUT, "Now, pipe some URL's into > %s\n", fifo); 400 g->fifo_event = event_new(g->evbase, sockfd, EV_READ|EV_PERSIST, fifo_cb, g); 401 event_add(g->fifo_event, NULL); 402 return (0); 403 } 404 405 static void clean_fifo(GlobalInfo *g) 406 { 407 event_free(g->fifo_event); 408 fclose(g->input); 409 unlink(fifo); 410 } 411 412 int main(int argc, char **argv) 413 { 414 GlobalInfo g; 415 (void)argc; 416 (void)argv; 417 418 memset(&g, 0, sizeof(GlobalInfo)); 419 g.evbase = event_base_new(); 420 init_fifo(&g); 421 g.multi = curl_multi_init(); 422 g.timer_event = evtimer_new(g.evbase, timer_cb, &g); 423 424 /* setup the generic multi interface options we want */ 425 curl_multi_setopt(g.multi, CURLMOPT_SOCKETFUNCTION, sock_cb); 426 curl_multi_setopt(g.multi, CURLMOPT_SOCKETDATA, &g); 427 curl_multi_setopt(g.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb); 428 curl_multi_setopt(g.multi, CURLMOPT_TIMERDATA, &g); 429 430 /* we don't call any curl_multi_socket*() function yet as we have no handles 431 added! */ 432 433 event_base_dispatch(g.evbase); 434 435 /* this, of course, won't get called since only way to stop this program is 436 via ctrl-C, but it is here to show how cleanup /would/ be done. */ 437 clean_fifo(&g); 438 event_free(g.timer_event); 439 event_base_free(g.evbase); 440 curl_multi_cleanup(g.multi); 441 return 0; 442 } 443
