Home | History | Annotate | Download | only in libevent
      1 /*
      2  * Copyright (c) 2009-2012 Niels Provos and Nick Mathewson
      3  *
      4  * Redistribution and use in source and binary forms, with or without
      5  * modification, are permitted provided that the following conditions
      6  * are met:
      7  * 1. Redistributions of source code must retain the above copyright
      8  *    notice, this list of conditions and the following disclaimer.
      9  * 2. Redistributions in binary form must reproduce the above copyright
     10  *    notice, this list of conditions and the following disclaimer in the
     11  *    documentation and/or other materials provided with the distribution.
     12  * 3. The name of the author may not be used to endorse or promote products
     13  *    derived from this software without specific prior written permission.
     14  *
     15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     25  */
     26 
     27 // Get rid of OSX 10.7 and greater deprecation warnings.
     28 #if defined(__APPLE__) && defined(__clang__)
     29 #pragma clang diagnostic ignored "-Wdeprecated-declarations"
     30 #endif
     31 
     32 #include "event2/event-config.h"
     33 #include "evconfig-private.h"
     34 
     35 #include <sys/types.h>
     36 
     37 #ifdef EVENT__HAVE_SYS_TIME_H
     38 #include <sys/time.h>
     39 #endif
     40 
     41 #include <errno.h>
     42 #include <stdio.h>
     43 #include <stdlib.h>
     44 #include <string.h>
     45 #ifdef EVENT__HAVE_STDARG_H
     46 #include <stdarg.h>
     47 #endif
     48 #ifdef EVENT__HAVE_UNISTD_H
     49 #include <unistd.h>
     50 #endif
     51 
     52 #ifdef _WIN32
     53 #include <winsock2.h>
     54 #endif
     55 
     56 #include "event2/bufferevent.h"
     57 #include "event2/bufferevent_struct.h"
     58 #include "event2/bufferevent_ssl.h"
     59 #include "event2/buffer.h"
     60 #include "event2/event.h"
     61 
     62 #include "mm-internal.h"
     63 #include "bufferevent-internal.h"
     64 #include "log-internal.h"
     65 
     66 #include <openssl/bio.h>
     67 #include <openssl/ssl.h>
     68 #include <openssl/err.h>
     69 #include "openssl-compat.h"
     70 
     71 /*
     72  * Define an OpenSSL bio that targets a bufferevent.
     73  */
     74 
     75 /* --------------------
     76    A BIO is an OpenSSL abstraction that handles reading and writing data.  The
     77    library will happily speak SSL over anything that implements a BIO
     78    interface.
     79 
     80    Here we define a BIO implementation that directs its output to a
     81    bufferevent.  We'll want to use this only when none of OpenSSL's built-in
     82    IO mechanisms work for us.
     83    -------------------- */
     84 
     85 /* every BIO type needs its own integer type value. */
     86 #define BIO_TYPE_LIBEVENT 57
     87 /* ???? Arguably, we should set BIO_TYPE_FILTER or BIO_TYPE_SOURCE_SINK on
     88  * this. */
     89 
     90 #if 0
     91 static void
     92 print_err(int val)
     93 {
     94 	int err;
     95 	printf("Error was %d\n", val);
     96 
     97 	while ((err = ERR_get_error())) {
     98 		const char *msg = (const char*)ERR_reason_error_string(err);
     99 		const char *lib = (const char*)ERR_lib_error_string(err);
    100 		const char *func = (const char*)ERR_func_error_string(err);
    101 
    102 		printf("%s in %s %s\n", msg, lib, func);
    103 	}
    104 }
    105 #else
    106 #define print_err(v) ((void)0)
    107 #endif
    108 
    109 /* Called to initialize a new BIO */
    110 static int
    111 bio_bufferevent_new(BIO *b)
    112 {
    113 	BIO_set_init(b, 0);
    114 	BIO_set_data(b, NULL); /* We'll be putting the bufferevent in this field.*/
    115 	return 1;
    116 }
    117 
    118 /* Called to uninitialize the BIO. */
    119 static int
    120 bio_bufferevent_free(BIO *b)
    121 {
    122 	if (!b)
    123 		return 0;
    124 	if (BIO_get_shutdown(b)) {
    125 		if (BIO_get_init(b) && BIO_get_data(b))
    126 			bufferevent_free(BIO_get_data(b));
    127 		BIO_free(b);
    128 	}
    129 	return 1;
    130 }
    131 
    132 /* Called to extract data from the BIO. */
    133 static int
    134 bio_bufferevent_read(BIO *b, char *out, int outlen)
    135 {
    136 	int r = 0;
    137 	struct evbuffer *input;
    138 
    139 	BIO_clear_retry_flags(b);
    140 
    141 	if (!out)
    142 		return 0;
    143 	if (!BIO_get_data(b))
    144 		return -1;
    145 
    146 	input = bufferevent_get_input(BIO_get_data(b));
    147 	if (evbuffer_get_length(input) == 0) {
    148 		/* If there's no data to read, say so. */
    149 		BIO_set_retry_read(b);
    150 		return -1;
    151 	} else {
    152 		r = evbuffer_remove(input, out, outlen);
    153 	}
    154 
    155 	return r;
    156 }
    157 
    158 /* Called to write data info the BIO */
    159 static int
    160 bio_bufferevent_write(BIO *b, const char *in, int inlen)
    161 {
    162 	struct bufferevent *bufev = BIO_get_data(b);
    163 	struct evbuffer *output;
    164 	size_t outlen;
    165 
    166 	BIO_clear_retry_flags(b);
    167 
    168 	if (!BIO_get_data(b))
    169 		return -1;
    170 
    171 	output = bufferevent_get_output(bufev);
    172 	outlen = evbuffer_get_length(output);
    173 
    174 	/* Copy only as much data onto the output buffer as can fit under the
    175 	 * high-water mark. */
    176 	if (bufev->wm_write.high && bufev->wm_write.high <= (outlen+inlen)) {
    177 		if (bufev->wm_write.high <= outlen) {
    178 			/* If no data can fit, we'll need to retry later. */
    179 			BIO_set_retry_write(b);
    180 			return -1;
    181 		}
    182 		inlen = bufev->wm_write.high - outlen;
    183 	}
    184 
    185 	EVUTIL_ASSERT(inlen > 0);
    186 	evbuffer_add(output, in, inlen);
    187 	return inlen;
    188 }
    189 
    190 /* Called to handle various requests */
    191 static long
    192 bio_bufferevent_ctrl(BIO *b, int cmd, long num, void *ptr)
    193 {
    194 	struct bufferevent *bufev = BIO_get_data(b);
    195 	long ret = 1;
    196 
    197 	switch (cmd) {
    198 	case BIO_CTRL_GET_CLOSE:
    199 		ret = BIO_get_shutdown(b);
    200 		break;
    201 	case BIO_CTRL_SET_CLOSE:
    202 		BIO_set_shutdown(b, (int)num);
    203 		break;
    204 	case BIO_CTRL_PENDING:
    205 		ret = evbuffer_get_length(bufferevent_get_input(bufev)) != 0;
    206 		break;
    207 	case BIO_CTRL_WPENDING:
    208 		ret = evbuffer_get_length(bufferevent_get_output(bufev)) != 0;
    209 		break;
    210 	/* XXXX These two are given a special-case treatment because
    211 	 * of cargo-cultism.  I should come up with a better reason. */
    212 	case BIO_CTRL_DUP:
    213 	case BIO_CTRL_FLUSH:
    214 		ret = 1;
    215 		break;
    216 	default:
    217 		ret = 0;
    218 		break;
    219 	}
    220 	return ret;
    221 }
    222 
    223 /* Called to write a string to the BIO */
    224 static int
    225 bio_bufferevent_puts(BIO *b, const char *s)
    226 {
    227 	return bio_bufferevent_write(b, s, strlen(s));
    228 }
    229 
    230 /* Method table for the bufferevent BIO */
    231 static BIO_METHOD *methods_bufferevent;
    232 
    233 /* Return the method table for the bufferevents BIO */
    234 static BIO_METHOD *
    235 BIO_s_bufferevent(void)
    236 {
    237 	if (methods_bufferevent == NULL) {
    238 		methods_bufferevent = BIO_meth_new(BIO_TYPE_LIBEVENT, "bufferevent");
    239 		if (methods_bufferevent == NULL)
    240 			return NULL;
    241 		BIO_meth_set_write(methods_bufferevent, bio_bufferevent_write);
    242 		BIO_meth_set_read(methods_bufferevent, bio_bufferevent_read);
    243 		BIO_meth_set_puts(methods_bufferevent, bio_bufferevent_puts);
    244 		BIO_meth_set_ctrl(methods_bufferevent, bio_bufferevent_ctrl);
    245 		BIO_meth_set_create(methods_bufferevent, bio_bufferevent_new);
    246 		BIO_meth_set_destroy(methods_bufferevent, bio_bufferevent_free);
    247 	}
    248 	return methods_bufferevent;
    249 }
    250 
    251 /* Create a new BIO to wrap communication around a bufferevent.  If close_flag
    252  * is true, the bufferevent will be freed when the BIO is closed. */
    253 static BIO *
    254 BIO_new_bufferevent(struct bufferevent *bufferevent, int close_flag)
    255 {
    256 	BIO *result;
    257 	if (!bufferevent)
    258 		return NULL;
    259 	if (!(result = BIO_new(BIO_s_bufferevent())))
    260 		return NULL;
    261 	BIO_set_init(result, 1);
    262 	BIO_set_data(result, bufferevent);
    263 	BIO_set_shutdown(result, close_flag ? 1 : 0);
    264 	return result;
    265 }
    266 
    267 /* --------------------
    268    Now, here's the OpenSSL-based implementation of bufferevent.
    269 
    270    The implementation comes in two flavors: one that connects its SSL object
    271    to an underlying bufferevent using a BIO_bufferevent, and one that has the
    272    SSL object connect to a socket directly.  The latter should generally be
    273    faster, except on Windows, where your best bet is using a
    274    bufferevent_async.
    275 
    276    (OpenSSL supports many other BIO types, too.  But we can't use any unless
    277    we have a good way to get notified when they become readable/writable.)
    278    -------------------- */
    279 
    280 struct bio_data_counts {
    281 	unsigned long n_written;
    282 	unsigned long n_read;
    283 };
    284 
    285 struct bufferevent_openssl {
    286 	/* Shared fields with common bufferevent implementation code.
    287 	   If we were set up with an underlying bufferevent, we use the
    288 	   events here as timers only.  If we have an SSL, then we use
    289 	   the events as socket events.
    290 	 */
    291 	struct bufferevent_private bev;
    292 	/* An underlying bufferevent that we're directing our output to.
    293 	   If it's NULL, then we're connected to an fd, not an evbuffer. */
    294 	struct bufferevent *underlying;
    295 	/* The SSL object doing our encryption. */
    296 	SSL *ssl;
    297 
    298 	/* A callback that's invoked when data arrives on our outbuf so we
    299 	   know to write data to the SSL. */
    300 	struct evbuffer_cb_entry *outbuf_cb;
    301 
    302 	/* A count of how much data the bios have read/written total.  Used
    303 	   for rate-limiting. */
    304 	struct bio_data_counts counts;
    305 
    306 	/* If this value is greater than 0, then the last SSL_write blocked,
    307 	 * and we need to try it again with this many bytes. */
    308 	ev_ssize_t last_write;
    309 
    310 #define NUM_ERRORS 3
    311 	ev_uint32_t errors[NUM_ERRORS];
    312 
    313 	/* When we next get available space, we should say "read" instead of
    314 	   "write". This can happen if there's a renegotiation during a read
    315 	   operation. */
    316 	unsigned read_blocked_on_write : 1;
    317 	/* When we next get data, we should say "write" instead of "read". */
    318 	unsigned write_blocked_on_read : 1;
    319 	/* Treat TCP close before SSL close on SSL >= v3 as clean EOF. */
    320 	unsigned allow_dirty_shutdown : 1;
    321 	/* XXX */
    322 	unsigned n_errors : 2;
    323 
    324 	/* Are we currently connecting, accepting, or doing IO? */
    325 	unsigned state : 2;
    326 	/* If we reset fd, we sould reset state too */
    327 	unsigned old_state : 2;
    328 };
    329 
    330 static int be_openssl_enable(struct bufferevent *, short);
    331 static int be_openssl_disable(struct bufferevent *, short);
    332 static void be_openssl_unlink(struct bufferevent *);
    333 static void be_openssl_destruct(struct bufferevent *);
    334 static int be_openssl_adj_timeouts(struct bufferevent *);
    335 static int be_openssl_flush(struct bufferevent *bufev,
    336     short iotype, enum bufferevent_flush_mode mode);
    337 static int be_openssl_ctrl(struct bufferevent *, enum bufferevent_ctrl_op, union bufferevent_ctrl_data *);
    338 
    339 const struct bufferevent_ops bufferevent_ops_openssl = {
    340 	"ssl",
    341 	evutil_offsetof(struct bufferevent_openssl, bev.bev),
    342 	be_openssl_enable,
    343 	be_openssl_disable,
    344 	be_openssl_unlink,
    345 	be_openssl_destruct,
    346 	be_openssl_adj_timeouts,
    347 	be_openssl_flush,
    348 	be_openssl_ctrl,
    349 };
    350 
    351 /* Given a bufferevent, return a pointer to the bufferevent_openssl that
    352  * contains it, if any. */
    353 static inline struct bufferevent_openssl *
    354 upcast(struct bufferevent *bev)
    355 {
    356 	struct bufferevent_openssl *bev_o;
    357 	if (bev->be_ops != &bufferevent_ops_openssl)
    358 		return NULL;
    359 	bev_o = (void*)( ((char*)bev) -
    360 			 evutil_offsetof(struct bufferevent_openssl, bev.bev));
    361 	EVUTIL_ASSERT(bev_o->bev.bev.be_ops == &bufferevent_ops_openssl);
    362 	return bev_o;
    363 }
    364 
    365 static inline void
    366 put_error(struct bufferevent_openssl *bev_ssl, unsigned long err)
    367 {
    368 	if (bev_ssl->n_errors == NUM_ERRORS)
    369 		return;
    370 	/* The error type according to openssl is "unsigned long", but
    371 	   openssl never uses more than 32 bits of it.  It _can't_ use more
    372 	   than 32 bits of it, since it needs to report errors on systems
    373 	   where long is only 32 bits.
    374 	 */
    375 	bev_ssl->errors[bev_ssl->n_errors++] = (ev_uint32_t) err;
    376 }
    377 
    378 /* Have the base communications channel (either the underlying bufferevent or
    379  * ev_read and ev_write) start reading.  Take the read-blocked-on-write flag
    380  * into account. */
    381 static int
    382 start_reading(struct bufferevent_openssl *bev_ssl)
    383 {
    384 	if (bev_ssl->underlying) {
    385 		bufferevent_unsuspend_read_(bev_ssl->underlying,
    386 		    BEV_SUSPEND_FILT_READ);
    387 		return 0;
    388 	} else {
    389 		struct bufferevent *bev = &bev_ssl->bev.bev;
    390 		int r;
    391 		r = bufferevent_add_event_(&bev->ev_read, &bev->timeout_read);
    392 		if (r == 0 && bev_ssl->read_blocked_on_write)
    393 			r = bufferevent_add_event_(&bev->ev_write,
    394 			    &bev->timeout_write);
    395 		return r;
    396 	}
    397 }
    398 
    399 /* Have the base communications channel (either the underlying bufferevent or
    400  * ev_read and ev_write) start writing.  Take the write-blocked-on-read flag
    401  * into account. */
    402 static int
    403 start_writing(struct bufferevent_openssl *bev_ssl)
    404 {
    405 	int r = 0;
    406 	if (bev_ssl->underlying) {
    407 		if (bev_ssl->write_blocked_on_read) {
    408 			bufferevent_unsuspend_read_(bev_ssl->underlying,
    409 			    BEV_SUSPEND_FILT_READ);
    410 		}
    411 	} else {
    412 		struct bufferevent *bev = &bev_ssl->bev.bev;
    413 		r = bufferevent_add_event_(&bev->ev_write, &bev->timeout_write);
    414 		if (!r && bev_ssl->write_blocked_on_read)
    415 			r = bufferevent_add_event_(&bev->ev_read,
    416 			    &bev->timeout_read);
    417 	}
    418 	return r;
    419 }
    420 
    421 static void
    422 stop_reading(struct bufferevent_openssl *bev_ssl)
    423 {
    424 	if (bev_ssl->write_blocked_on_read)
    425 		return;
    426 	if (bev_ssl->underlying) {
    427 		bufferevent_suspend_read_(bev_ssl->underlying,
    428 		    BEV_SUSPEND_FILT_READ);
    429 	} else {
    430 		struct bufferevent *bev = &bev_ssl->bev.bev;
    431 		event_del(&bev->ev_read);
    432 	}
    433 }
    434 
    435 static void
    436 stop_writing(struct bufferevent_openssl *bev_ssl)
    437 {
    438 	if (bev_ssl->read_blocked_on_write)
    439 		return;
    440 	if (bev_ssl->underlying) {
    441 		bufferevent_unsuspend_read_(bev_ssl->underlying,
    442 		    BEV_SUSPEND_FILT_READ);
    443 	} else {
    444 		struct bufferevent *bev = &bev_ssl->bev.bev;
    445 		event_del(&bev->ev_write);
    446 	}
    447 }
    448 
    449 static int
    450 set_rbow(struct bufferevent_openssl *bev_ssl)
    451 {
    452 	if (!bev_ssl->underlying)
    453 		stop_reading(bev_ssl);
    454 	bev_ssl->read_blocked_on_write = 1;
    455 	return start_writing(bev_ssl);
    456 }
    457 
    458 static int
    459 set_wbor(struct bufferevent_openssl *bev_ssl)
    460 {
    461 	if (!bev_ssl->underlying)
    462 		stop_writing(bev_ssl);
    463 	bev_ssl->write_blocked_on_read = 1;
    464 	return start_reading(bev_ssl);
    465 }
    466 
    467 static int
    468 clear_rbow(struct bufferevent_openssl *bev_ssl)
    469 {
    470 	struct bufferevent *bev = &bev_ssl->bev.bev;
    471 	int r = 0;
    472 	bev_ssl->read_blocked_on_write = 0;
    473 	if (!(bev->enabled & EV_WRITE))
    474 		stop_writing(bev_ssl);
    475 	if (bev->enabled & EV_READ)
    476 		r = start_reading(bev_ssl);
    477 	return r;
    478 }
    479 
    480 
    481 static int
    482 clear_wbor(struct bufferevent_openssl *bev_ssl)
    483 {
    484 	struct bufferevent *bev = &bev_ssl->bev.bev;
    485 	int r = 0;
    486 	bev_ssl->write_blocked_on_read = 0;
    487 	if (!(bev->enabled & EV_READ))
    488 		stop_reading(bev_ssl);
    489 	if (bev->enabled & EV_WRITE)
    490 		r = start_writing(bev_ssl);
    491 	return r;
    492 }
    493 
    494 static void
    495 conn_closed(struct bufferevent_openssl *bev_ssl, int when, int errcode, int ret)
    496 {
    497 	int event = BEV_EVENT_ERROR;
    498 	int dirty_shutdown = 0;
    499 	unsigned long err;
    500 
    501 	switch (errcode) {
    502 	case SSL_ERROR_ZERO_RETURN:
    503 		/* Possibly a clean shutdown. */
    504 		if (SSL_get_shutdown(bev_ssl->ssl) & SSL_RECEIVED_SHUTDOWN)
    505 			event = BEV_EVENT_EOF;
    506 		else
    507 			dirty_shutdown = 1;
    508 		break;
    509 	case SSL_ERROR_SYSCALL:
    510 		/* IO error; possibly a dirty shutdown. */
    511 		if ((ret == 0 || ret == -1) && ERR_peek_error() == 0)
    512 			dirty_shutdown = 1;
    513 		break;
    514 	case SSL_ERROR_SSL:
    515 		/* Protocol error. */
    516 		break;
    517 	case SSL_ERROR_WANT_X509_LOOKUP:
    518 		/* XXXX handle this. */
    519 		break;
    520 	case SSL_ERROR_NONE:
    521 	case SSL_ERROR_WANT_READ:
    522 	case SSL_ERROR_WANT_WRITE:
    523 	case SSL_ERROR_WANT_CONNECT:
    524 	case SSL_ERROR_WANT_ACCEPT:
    525 	default:
    526 		/* should be impossible; treat as normal error. */
    527 		event_warnx("BUG: Unexpected OpenSSL error code %d", errcode);
    528 		break;
    529 	}
    530 
    531 	while ((err = ERR_get_error())) {
    532 		put_error(bev_ssl, err);
    533 	}
    534 
    535 	if (dirty_shutdown && bev_ssl->allow_dirty_shutdown)
    536 		event = BEV_EVENT_EOF;
    537 
    538 	stop_reading(bev_ssl);
    539 	stop_writing(bev_ssl);
    540 
    541 	/* when is BEV_EVENT_{READING|WRITING} */
    542 	event = when | event;
    543 	bufferevent_run_eventcb_(&bev_ssl->bev.bev, event, 0);
    544 }
    545 
    546 static void
    547 init_bio_counts(struct bufferevent_openssl *bev_ssl)
    548 {
    549 	BIO *rbio, *wbio;
    550 
    551 	wbio = SSL_get_wbio(bev_ssl->ssl);
    552 	bev_ssl->counts.n_written = wbio ? BIO_number_written(wbio) : 0;
    553 	rbio = SSL_get_rbio(bev_ssl->ssl);
    554 	bev_ssl->counts.n_read = rbio ? BIO_number_read(rbio) : 0;
    555 }
    556 
    557 static inline void
    558 decrement_buckets(struct bufferevent_openssl *bev_ssl)
    559 {
    560 	unsigned long num_w = BIO_number_written(SSL_get_wbio(bev_ssl->ssl));
    561 	unsigned long num_r = BIO_number_read(SSL_get_rbio(bev_ssl->ssl));
    562 	/* These next two subtractions can wrap around. That's okay. */
    563 	unsigned long w = num_w - bev_ssl->counts.n_written;
    564 	unsigned long r = num_r - bev_ssl->counts.n_read;
    565 	if (w)
    566 		bufferevent_decrement_write_buckets_(&bev_ssl->bev, w);
    567 	if (r)
    568 		bufferevent_decrement_read_buckets_(&bev_ssl->bev, r);
    569 	bev_ssl->counts.n_written = num_w;
    570 	bev_ssl->counts.n_read = num_r;
    571 }
    572 
    573 #define OP_MADE_PROGRESS 1
    574 #define OP_BLOCKED 2
    575 #define OP_ERR 4
    576 
    577 /* Return a bitmask of OP_MADE_PROGRESS (if we read anything); OP_BLOCKED (if
    578    we're now blocked); and OP_ERR (if an error occurred). */
    579 static int
    580 do_read(struct bufferevent_openssl *bev_ssl, int n_to_read) {
    581 	/* Requires lock */
    582 	struct bufferevent *bev = &bev_ssl->bev.bev;
    583 	struct evbuffer *input = bev->input;
    584 	int r, n, i, n_used = 0, atmost;
    585 	struct evbuffer_iovec space[2];
    586 	int result = 0;
    587 
    588 	if (bev_ssl->bev.read_suspended)
    589 		return 0;
    590 
    591 	atmost = bufferevent_get_read_max_(&bev_ssl->bev);
    592 	if (n_to_read > atmost)
    593 		n_to_read = atmost;
    594 
    595 	n = evbuffer_reserve_space(input, n_to_read, space, 2);
    596 	if (n < 0)
    597 		return OP_ERR;
    598 
    599 	for (i=0; i<n; ++i) {
    600 		if (bev_ssl->bev.read_suspended)
    601 			break;
    602 		ERR_clear_error();
    603 		r = SSL_read(bev_ssl->ssl, space[i].iov_base, space[i].iov_len);
    604 		if (r>0) {
    605 			result |= OP_MADE_PROGRESS;
    606 			if (bev_ssl->read_blocked_on_write)
    607 				if (clear_rbow(bev_ssl) < 0)
    608 					return OP_ERR | result;
    609 			++n_used;
    610 			space[i].iov_len = r;
    611 			decrement_buckets(bev_ssl);
    612 		} else {
    613 			int err = SSL_get_error(bev_ssl->ssl, r);
    614 			print_err(err);
    615 			switch (err) {
    616 			case SSL_ERROR_WANT_READ:
    617 				/* Can't read until underlying has more data. */
    618 				if (bev_ssl->read_blocked_on_write)
    619 					if (clear_rbow(bev_ssl) < 0)
    620 						return OP_ERR | result;
    621 				break;
    622 			case SSL_ERROR_WANT_WRITE:
    623 				/* This read operation requires a write, and the
    624 				 * underlying is full */
    625 				if (!bev_ssl->read_blocked_on_write)
    626 					if (set_rbow(bev_ssl) < 0)
    627 						return OP_ERR | result;
    628 				break;
    629 			default:
    630 				conn_closed(bev_ssl, BEV_EVENT_READING, err, r);
    631 				break;
    632 			}
    633 			result |= OP_BLOCKED;
    634 			break; /* out of the loop */
    635 		}
    636 	}
    637 
    638 	if (n_used) {
    639 		evbuffer_commit_space(input, space, n_used);
    640 		if (bev_ssl->underlying)
    641 			BEV_RESET_GENERIC_READ_TIMEOUT(bev);
    642 	}
    643 
    644 	return result;
    645 }
    646 
    647 /* Return a bitmask of OP_MADE_PROGRESS (if we wrote anything); OP_BLOCKED (if
    648    we're now blocked); and OP_ERR (if an error occurred). */
    649 static int
    650 do_write(struct bufferevent_openssl *bev_ssl, int atmost)
    651 {
    652 	int i, r, n, n_written = 0;
    653 	struct bufferevent *bev = &bev_ssl->bev.bev;
    654 	struct evbuffer *output = bev->output;
    655 	struct evbuffer_iovec space[8];
    656 	int result = 0;
    657 
    658 	if (bev_ssl->last_write > 0)
    659 		atmost = bev_ssl->last_write;
    660 	else
    661 		atmost = bufferevent_get_write_max_(&bev_ssl->bev);
    662 
    663 	n = evbuffer_peek(output, atmost, NULL, space, 8);
    664 	if (n < 0)
    665 		return OP_ERR | result;
    666 
    667 	if (n > 8)
    668 		n = 8;
    669 	for (i=0; i < n; ++i) {
    670 		if (bev_ssl->bev.write_suspended)
    671 			break;
    672 
    673 		/* SSL_write will (reasonably) return 0 if we tell it to
    674 		   send 0 data.  Skip this case so we don't interpret the
    675 		   result as an error */
    676 		if (space[i].iov_len == 0)
    677 			continue;
    678 
    679 		ERR_clear_error();
    680 		r = SSL_write(bev_ssl->ssl, space[i].iov_base,
    681 		    space[i].iov_len);
    682 		if (r > 0) {
    683 			result |= OP_MADE_PROGRESS;
    684 			if (bev_ssl->write_blocked_on_read)
    685 				if (clear_wbor(bev_ssl) < 0)
    686 					return OP_ERR | result;
    687 			n_written += r;
    688 			bev_ssl->last_write = -1;
    689 			decrement_buckets(bev_ssl);
    690 		} else {
    691 			int err = SSL_get_error(bev_ssl->ssl, r);
    692 			print_err(err);
    693 			switch (err) {
    694 			case SSL_ERROR_WANT_WRITE:
    695 				/* Can't read until underlying has more data. */
    696 				if (bev_ssl->write_blocked_on_read)
    697 					if (clear_wbor(bev_ssl) < 0)
    698 						return OP_ERR | result;
    699 				bev_ssl->last_write = space[i].iov_len;
    700 				break;
    701 			case SSL_ERROR_WANT_READ:
    702 				/* This read operation requires a write, and the
    703 				 * underlying is full */
    704 				if (!bev_ssl->write_blocked_on_read)
    705 					if (set_wbor(bev_ssl) < 0)
    706 						return OP_ERR | result;
    707 				bev_ssl->last_write = space[i].iov_len;
    708 				break;
    709 			default:
    710 				conn_closed(bev_ssl, BEV_EVENT_WRITING, err, r);
    711 				bev_ssl->last_write = -1;
    712 				break;
    713 			}
    714 			result |= OP_BLOCKED;
    715 			break;
    716 		}
    717 	}
    718 	if (n_written) {
    719 		evbuffer_drain(output, n_written);
    720 		if (bev_ssl->underlying)
    721 			BEV_RESET_GENERIC_WRITE_TIMEOUT(bev);
    722 
    723 		bufferevent_trigger_nolock_(bev, EV_WRITE, BEV_OPT_DEFER_CALLBACKS);
    724 	}
    725 	return result;
    726 }
    727 
    728 #define WRITE_FRAME 15000
    729 
    730 #define READ_DEFAULT 4096
    731 
    732 /* Try to figure out how many bytes to read; return 0 if we shouldn't be
    733  * reading. */
    734 static int
    735 bytes_to_read(struct bufferevent_openssl *bev)
    736 {
    737 	struct evbuffer *input = bev->bev.bev.input;
    738 	struct event_watermark *wm = &bev->bev.bev.wm_read;
    739 	int result = READ_DEFAULT;
    740 	ev_ssize_t limit;
    741 	/* XXX 99% of this is generic code that nearly all bufferevents will
    742 	 * want. */
    743 
    744 	if (bev->write_blocked_on_read) {
    745 		return 0;
    746 	}
    747 
    748 	if (! (bev->bev.bev.enabled & EV_READ)) {
    749 		return 0;
    750 	}
    751 
    752 	if (bev->bev.read_suspended) {
    753 		return 0;
    754 	}
    755 
    756 	if (wm->high) {
    757 		if (evbuffer_get_length(input) >= wm->high) {
    758 			return 0;
    759 		}
    760 
    761 		result = wm->high - evbuffer_get_length(input);
    762 	} else {
    763 		result = READ_DEFAULT;
    764 	}
    765 
    766 	/* Respect the rate limit */
    767 	limit = bufferevent_get_read_max_(&bev->bev);
    768 	if (result > limit) {
    769 		result = limit;
    770 	}
    771 
    772 	return result;
    773 }
    774 
    775 
    776 /* Things look readable.  If write is blocked on read, write till it isn't.
    777  * Read from the underlying buffer until we block or we hit our high-water
    778  * mark.
    779  */
    780 static void
    781 consider_reading(struct bufferevent_openssl *bev_ssl)
    782 {
    783 	int r;
    784 	int n_to_read;
    785 	int all_result_flags = 0;
    786 
    787 	while (bev_ssl->write_blocked_on_read) {
    788 		r = do_write(bev_ssl, WRITE_FRAME);
    789 		if (r & (OP_BLOCKED|OP_ERR))
    790 			break;
    791 	}
    792 	if (bev_ssl->write_blocked_on_read)
    793 		return;
    794 
    795 	n_to_read = bytes_to_read(bev_ssl);
    796 
    797 	while (n_to_read) {
    798 		r = do_read(bev_ssl, n_to_read);
    799 		all_result_flags |= r;
    800 
    801 		if (r & (OP_BLOCKED|OP_ERR))
    802 			break;
    803 
    804 		if (bev_ssl->bev.read_suspended)
    805 			break;
    806 
    807 		/* Read all pending data.  This won't hit the network
    808 		 * again, and will (most importantly) put us in a state
    809 		 * where we don't need to read anything else until the
    810 		 * socket is readable again.  It'll potentially make us
    811 		 * overrun our read high-watermark (somewhat
    812 		 * regrettable).  The damage to the rate-limit has
    813 		 * already been done, since OpenSSL went and read a
    814 		 * whole SSL record anyway. */
    815 		n_to_read = SSL_pending(bev_ssl->ssl);
    816 
    817 		/* XXX This if statement is actually a bad bug, added to avoid
    818 		 * XXX a worse bug.
    819 		 *
    820 		 * The bad bug: It can potentially cause resource unfairness
    821 		 * by reading too much data from the underlying bufferevent;
    822 		 * it can potentially cause read looping if the underlying
    823 		 * bufferevent is a bufferevent_pair and deferred callbacks
    824 		 * aren't used.
    825 		 *
    826 		 * The worse bug: If we didn't do this, then we would
    827 		 * potentially not read any more from bev_ssl->underlying
    828 		 * until more data arrived there, which could lead to us
    829 		 * waiting forever.
    830 		 */
    831 		if (!n_to_read && bev_ssl->underlying)
    832 			n_to_read = bytes_to_read(bev_ssl);
    833 	}
    834 
    835 	if (all_result_flags & OP_MADE_PROGRESS) {
    836 		struct bufferevent *bev = &bev_ssl->bev.bev;
    837 
    838 		bufferevent_trigger_nolock_(bev, EV_READ, 0);
    839 	}
    840 
    841 	if (!bev_ssl->underlying) {
    842 		/* Should be redundant, but let's avoid busy-looping */
    843 		if (bev_ssl->bev.read_suspended ||
    844 		    !(bev_ssl->bev.bev.enabled & EV_READ)) {
    845 			event_del(&bev_ssl->bev.bev.ev_read);
    846 		}
    847 	}
    848 }
    849 
    850 static void
    851 consider_writing(struct bufferevent_openssl *bev_ssl)
    852 {
    853 	int r;
    854 	struct evbuffer *output = bev_ssl->bev.bev.output;
    855 	struct evbuffer *target = NULL;
    856 	struct event_watermark *wm = NULL;
    857 
    858 	while (bev_ssl->read_blocked_on_write) {
    859 		r = do_read(bev_ssl, 1024); /* XXXX 1024 is a hack */
    860 		if (r & OP_MADE_PROGRESS) {
    861 			struct bufferevent *bev = &bev_ssl->bev.bev;
    862 
    863 			bufferevent_trigger_nolock_(bev, EV_READ, 0);
    864 		}
    865 		if (r & (OP_ERR|OP_BLOCKED))
    866 			break;
    867 	}
    868 	if (bev_ssl->read_blocked_on_write)
    869 		return;
    870 	if (bev_ssl->underlying) {
    871 		target = bev_ssl->underlying->output;
    872 		wm = &bev_ssl->underlying->wm_write;
    873 	}
    874 	while ((bev_ssl->bev.bev.enabled & EV_WRITE) &&
    875 	    (! bev_ssl->bev.write_suspended) &&
    876 	    evbuffer_get_length(output) &&
    877 	    (!target || (! wm->high || evbuffer_get_length(target) < wm->high))) {
    878 		int n_to_write;
    879 		if (wm && wm->high)
    880 			n_to_write = wm->high - evbuffer_get_length(target);
    881 		else
    882 			n_to_write = WRITE_FRAME;
    883 		r = do_write(bev_ssl, n_to_write);
    884 		if (r & (OP_BLOCKED|OP_ERR))
    885 			break;
    886 	}
    887 
    888 	if (!bev_ssl->underlying) {
    889 		if (evbuffer_get_length(output) == 0) {
    890 			event_del(&bev_ssl->bev.bev.ev_write);
    891 		} else if (bev_ssl->bev.write_suspended ||
    892 		    !(bev_ssl->bev.bev.enabled & EV_WRITE)) {
    893 			/* Should be redundant, but let's avoid busy-looping */
    894 			event_del(&bev_ssl->bev.bev.ev_write);
    895 		}
    896 	}
    897 }
    898 
    899 static void
    900 be_openssl_readcb(struct bufferevent *bev_base, void *ctx)
    901 {
    902 	struct bufferevent_openssl *bev_ssl = ctx;
    903 	consider_reading(bev_ssl);
    904 }
    905 
    906 static void
    907 be_openssl_writecb(struct bufferevent *bev_base, void *ctx)
    908 {
    909 	struct bufferevent_openssl *bev_ssl = ctx;
    910 	consider_writing(bev_ssl);
    911 }
    912 
    913 static void
    914 be_openssl_eventcb(struct bufferevent *bev_base, short what, void *ctx)
    915 {
    916 	struct bufferevent_openssl *bev_ssl = ctx;
    917 	int event = 0;
    918 
    919 	if (what & BEV_EVENT_EOF) {
    920 		if (bev_ssl->allow_dirty_shutdown)
    921 			event = BEV_EVENT_EOF;
    922 		else
    923 			event = BEV_EVENT_ERROR;
    924 	} else if (what & BEV_EVENT_TIMEOUT) {
    925 		/* We sure didn't set this.  Propagate it to the user. */
    926 		event = what;
    927 	} else if (what & BEV_EVENT_ERROR) {
    928 		/* An error occurred on the connection.  Propagate it to the user. */
    929 		event = what;
    930 	} else if (what & BEV_EVENT_CONNECTED) {
    931 		/* Ignore it.  We're saying SSL_connect() already, which will
    932 		   eat it. */
    933 	}
    934 	if (event)
    935 		bufferevent_run_eventcb_(&bev_ssl->bev.bev, event, 0);
    936 }
    937 
    938 static void
    939 be_openssl_readeventcb(evutil_socket_t fd, short what, void *ptr)
    940 {
    941 	struct bufferevent_openssl *bev_ssl = ptr;
    942 	bufferevent_incref_and_lock_(&bev_ssl->bev.bev);
    943 	if (what == EV_TIMEOUT) {
    944 		bufferevent_run_eventcb_(&bev_ssl->bev.bev,
    945 		    BEV_EVENT_TIMEOUT|BEV_EVENT_READING, 0);
    946 	} else {
    947 		consider_reading(bev_ssl);
    948 	}
    949 	bufferevent_decref_and_unlock_(&bev_ssl->bev.bev);
    950 }
    951 
    952 static void
    953 be_openssl_writeeventcb(evutil_socket_t fd, short what, void *ptr)
    954 {
    955 	struct bufferevent_openssl *bev_ssl = ptr;
    956 	bufferevent_incref_and_lock_(&bev_ssl->bev.bev);
    957 	if (what == EV_TIMEOUT) {
    958 		bufferevent_run_eventcb_(&bev_ssl->bev.bev,
    959 		    BEV_EVENT_TIMEOUT|BEV_EVENT_WRITING, 0);
    960 	} else {
    961 		consider_writing(bev_ssl);
    962 	}
    963 	bufferevent_decref_and_unlock_(&bev_ssl->bev.bev);
    964 }
    965 
    966 static int
    967 be_openssl_auto_fd(struct bufferevent_openssl *bev_ssl, int fd)
    968 {
    969 	if (!bev_ssl->underlying) {
    970 		struct bufferevent *bev = &bev_ssl->bev.bev;
    971 		if (event_initialized(&bev->ev_read) && fd < 0) {
    972 			fd = event_get_fd(&bev->ev_read);
    973 		}
    974 	}
    975 	return fd;
    976 }
    977 
    978 static int
    979 set_open_callbacks(struct bufferevent_openssl *bev_ssl, evutil_socket_t fd)
    980 {
    981 	if (bev_ssl->underlying) {
    982 		bufferevent_setcb(bev_ssl->underlying,
    983 		    be_openssl_readcb, be_openssl_writecb, be_openssl_eventcb,
    984 		    bev_ssl);
    985 		return 0;
    986 	} else {
    987 		struct bufferevent *bev = &bev_ssl->bev.bev;
    988 		int rpending=0, wpending=0, r1=0, r2=0;
    989 
    990 		if (event_initialized(&bev->ev_read)) {
    991 			rpending = event_pending(&bev->ev_read, EV_READ, NULL);
    992 			wpending = event_pending(&bev->ev_write, EV_WRITE, NULL);
    993 
    994 			event_del(&bev->ev_read);
    995 			event_del(&bev->ev_write);
    996 		}
    997 
    998 		event_assign(&bev->ev_read, bev->ev_base, fd,
    999 		    EV_READ|EV_PERSIST|EV_FINALIZE,
   1000 		    be_openssl_readeventcb, bev_ssl);
   1001 		event_assign(&bev->ev_write, bev->ev_base, fd,
   1002 		    EV_WRITE|EV_PERSIST|EV_FINALIZE,
   1003 		    be_openssl_writeeventcb, bev_ssl);
   1004 
   1005 		if (rpending)
   1006 			r1 = bufferevent_add_event_(&bev->ev_read, &bev->timeout_read);
   1007 		if (wpending)
   1008 			r2 = bufferevent_add_event_(&bev->ev_write, &bev->timeout_write);
   1009 
   1010 		return (r1<0 || r2<0) ? -1 : 0;
   1011 	}
   1012 }
   1013 
   1014 static int
   1015 do_handshake(struct bufferevent_openssl *bev_ssl)
   1016 {
   1017 	int r;
   1018 
   1019 	switch (bev_ssl->state) {
   1020 	default:
   1021 	case BUFFEREVENT_SSL_OPEN:
   1022 		EVUTIL_ASSERT(0);
   1023 		return -1;
   1024 	case BUFFEREVENT_SSL_CONNECTING:
   1025 	case BUFFEREVENT_SSL_ACCEPTING:
   1026 		ERR_clear_error();
   1027 		r = SSL_do_handshake(bev_ssl->ssl);
   1028 		break;
   1029 	}
   1030 	decrement_buckets(bev_ssl);
   1031 
   1032 	if (r==1) {
   1033 		int fd = event_get_fd(&bev_ssl->bev.bev.ev_read);
   1034 		/* We're done! */
   1035 		bev_ssl->state = BUFFEREVENT_SSL_OPEN;
   1036 		set_open_callbacks(bev_ssl, fd); /* XXXX handle failure */
   1037 		/* Call do_read and do_write as needed */
   1038 		bufferevent_enable(&bev_ssl->bev.bev, bev_ssl->bev.bev.enabled);
   1039 		bufferevent_run_eventcb_(&bev_ssl->bev.bev,
   1040 		    BEV_EVENT_CONNECTED, 0);
   1041 		return 1;
   1042 	} else {
   1043 		int err = SSL_get_error(bev_ssl->ssl, r);
   1044 		print_err(err);
   1045 		switch (err) {
   1046 		case SSL_ERROR_WANT_WRITE:
   1047 			stop_reading(bev_ssl);
   1048 			return start_writing(bev_ssl);
   1049 		case SSL_ERROR_WANT_READ:
   1050 			stop_writing(bev_ssl);
   1051 			return start_reading(bev_ssl);
   1052 		default:
   1053 			conn_closed(bev_ssl, BEV_EVENT_READING, err, r);
   1054 			return -1;
   1055 		}
   1056 	}
   1057 }
   1058 
   1059 static void
   1060 be_openssl_handshakecb(struct bufferevent *bev_base, void *ctx)
   1061 {
   1062 	struct bufferevent_openssl *bev_ssl = ctx;
   1063 	do_handshake(bev_ssl);/* XXX handle failure */
   1064 }
   1065 
   1066 static void
   1067 be_openssl_handshakeeventcb(evutil_socket_t fd, short what, void *ptr)
   1068 {
   1069 	struct bufferevent_openssl *bev_ssl = ptr;
   1070 
   1071 	bufferevent_incref_and_lock_(&bev_ssl->bev.bev);
   1072 	if (what & EV_TIMEOUT) {
   1073 		bufferevent_run_eventcb_(&bev_ssl->bev.bev, BEV_EVENT_TIMEOUT, 0);
   1074 	} else
   1075 		do_handshake(bev_ssl);/* XXX handle failure */
   1076 	bufferevent_decref_and_unlock_(&bev_ssl->bev.bev);
   1077 }
   1078 
   1079 static int
   1080 set_handshake_callbacks(struct bufferevent_openssl *bev_ssl, evutil_socket_t fd)
   1081 {
   1082 	if (bev_ssl->underlying) {
   1083 		bufferevent_setcb(bev_ssl->underlying,
   1084 		    be_openssl_handshakecb, be_openssl_handshakecb,
   1085 		    be_openssl_eventcb,
   1086 		    bev_ssl);
   1087 
   1088 		if (fd < 0)
   1089 			return 0;
   1090 
   1091 		if (bufferevent_setfd(bev_ssl->underlying, fd))
   1092 			return 1;
   1093 
   1094 		return do_handshake(bev_ssl);
   1095 	} else {
   1096 		struct bufferevent *bev = &bev_ssl->bev.bev;
   1097 
   1098 		if (event_initialized(&bev->ev_read)) {
   1099 			event_del(&bev->ev_read);
   1100 			event_del(&bev->ev_write);
   1101 		}
   1102 
   1103 		event_assign(&bev->ev_read, bev->ev_base, fd,
   1104 		    EV_READ|EV_PERSIST|EV_FINALIZE,
   1105 		    be_openssl_handshakeeventcb, bev_ssl);
   1106 		event_assign(&bev->ev_write, bev->ev_base, fd,
   1107 		    EV_WRITE|EV_PERSIST|EV_FINALIZE,
   1108 		    be_openssl_handshakeeventcb, bev_ssl);
   1109 		if (fd >= 0)
   1110 			bufferevent_enable(bev, bev->enabled);
   1111 		return 0;
   1112 	}
   1113 }
   1114 
   1115 int
   1116 bufferevent_ssl_renegotiate(struct bufferevent *bev)
   1117 {
   1118 	struct bufferevent_openssl *bev_ssl = upcast(bev);
   1119 	if (!bev_ssl)
   1120 		return -1;
   1121 	if (SSL_renegotiate(bev_ssl->ssl) < 0)
   1122 		return -1;
   1123 	bev_ssl->state = BUFFEREVENT_SSL_CONNECTING;
   1124 	if (set_handshake_callbacks(bev_ssl, be_openssl_auto_fd(bev_ssl, -1)) < 0)
   1125 		return -1;
   1126 	if (!bev_ssl->underlying)
   1127 		return do_handshake(bev_ssl);
   1128 	return 0;
   1129 }
   1130 
   1131 static void
   1132 be_openssl_outbuf_cb(struct evbuffer *buf,
   1133     const struct evbuffer_cb_info *cbinfo, void *arg)
   1134 {
   1135 	struct bufferevent_openssl *bev_ssl = arg;
   1136 	int r = 0;
   1137 	/* XXX need to hold a reference here. */
   1138 
   1139 	if (cbinfo->n_added && bev_ssl->state == BUFFEREVENT_SSL_OPEN) {
   1140 		if (cbinfo->orig_size == 0)
   1141 			r = bufferevent_add_event_(&bev_ssl->bev.bev.ev_write,
   1142 			    &bev_ssl->bev.bev.timeout_write);
   1143 
   1144 		if (bev_ssl->underlying)
   1145 			consider_writing(bev_ssl);
   1146 	}
   1147 	/* XXX Handle r < 0 */
   1148 	(void)r;
   1149 }
   1150 
   1151 
   1152 static int
   1153 be_openssl_enable(struct bufferevent *bev, short events)
   1154 {
   1155 	struct bufferevent_openssl *bev_ssl = upcast(bev);
   1156 	int r1 = 0, r2 = 0;
   1157 
   1158 	if (events & EV_READ)
   1159 		r1 = start_reading(bev_ssl);
   1160 	if (events & EV_WRITE)
   1161 		r2 = start_writing(bev_ssl);
   1162 
   1163 	if (bev_ssl->underlying) {
   1164 		if (events & EV_READ)
   1165 			BEV_RESET_GENERIC_READ_TIMEOUT(bev);
   1166 		if (events & EV_WRITE)
   1167 			BEV_RESET_GENERIC_WRITE_TIMEOUT(bev);
   1168 
   1169 		if (events & EV_READ)
   1170 			consider_reading(bev_ssl);
   1171 		if (events & EV_WRITE)
   1172 			consider_writing(bev_ssl);
   1173 	}
   1174 	return (r1 < 0 || r2 < 0) ? -1 : 0;
   1175 }
   1176 
   1177 static int
   1178 be_openssl_disable(struct bufferevent *bev, short events)
   1179 {
   1180 	struct bufferevent_openssl *bev_ssl = upcast(bev);
   1181 
   1182 	if (events & EV_READ)
   1183 		stop_reading(bev_ssl);
   1184 	if (events & EV_WRITE)
   1185 		stop_writing(bev_ssl);
   1186 
   1187 	if (bev_ssl->underlying) {
   1188 		if (events & EV_READ)
   1189 			BEV_DEL_GENERIC_READ_TIMEOUT(bev);
   1190 		if (events & EV_WRITE)
   1191 			BEV_DEL_GENERIC_WRITE_TIMEOUT(bev);
   1192 	}
   1193 	return 0;
   1194 }
   1195 
   1196 static void
   1197 be_openssl_unlink(struct bufferevent *bev)
   1198 {
   1199 	struct bufferevent_openssl *bev_ssl = upcast(bev);
   1200 
   1201 	if (bev_ssl->bev.options & BEV_OPT_CLOSE_ON_FREE) {
   1202 		if (bev_ssl->underlying) {
   1203 			if (BEV_UPCAST(bev_ssl->underlying)->refcnt < 2) {
   1204 				event_warnx("BEV_OPT_CLOSE_ON_FREE set on an "
   1205 				    "bufferevent with too few references");
   1206 			} else {
   1207 				bufferevent_free(bev_ssl->underlying);
   1208 				/* We still have a reference to it, via our
   1209 				 * BIO. So we don't drop this. */
   1210 				// bev_ssl->underlying = NULL;
   1211 			}
   1212 		}
   1213 	} else {
   1214 		if (bev_ssl->underlying) {
   1215 			if (bev_ssl->underlying->errorcb == be_openssl_eventcb)
   1216 				bufferevent_setcb(bev_ssl->underlying,
   1217 				    NULL,NULL,NULL,NULL);
   1218 			bufferevent_unsuspend_read_(bev_ssl->underlying,
   1219 			    BEV_SUSPEND_FILT_READ);
   1220 		}
   1221 	}
   1222 }
   1223 
   1224 static void
   1225 be_openssl_destruct(struct bufferevent *bev)
   1226 {
   1227 	struct bufferevent_openssl *bev_ssl = upcast(bev);
   1228 
   1229 	if (bev_ssl->bev.options & BEV_OPT_CLOSE_ON_FREE) {
   1230 		if (! bev_ssl->underlying) {
   1231 			evutil_socket_t fd = -1;
   1232 			BIO *bio = SSL_get_wbio(bev_ssl->ssl);
   1233 			if (bio)
   1234 				fd = BIO_get_fd(bio, NULL);
   1235 			if (fd >= 0)
   1236 				evutil_closesocket(fd);
   1237 		}
   1238 		SSL_free(bev_ssl->ssl);
   1239 	}
   1240 }
   1241 
   1242 static int
   1243 be_openssl_adj_timeouts(struct bufferevent *bev)
   1244 {
   1245 	struct bufferevent_openssl *bev_ssl = upcast(bev);
   1246 
   1247 	if (bev_ssl->underlying) {
   1248 		return bufferevent_generic_adj_timeouts_(bev);
   1249 	} else {
   1250 		return bufferevent_generic_adj_existing_timeouts_(bev);
   1251 	}
   1252 }
   1253 
   1254 static int
   1255 be_openssl_flush(struct bufferevent *bufev,
   1256     short iotype, enum bufferevent_flush_mode mode)
   1257 {
   1258 	/* XXXX Implement this. */
   1259 	return 0;
   1260 }
   1261 
   1262 static int
   1263 be_openssl_set_fd(struct bufferevent_openssl *bev_ssl,
   1264     enum bufferevent_ssl_state state, int fd)
   1265 {
   1266 	bev_ssl->state = state;
   1267 
   1268 	switch (state) {
   1269 	case BUFFEREVENT_SSL_ACCEPTING:
   1270 		SSL_set_accept_state(bev_ssl->ssl);
   1271 		if (set_handshake_callbacks(bev_ssl, fd) < 0)
   1272 			return -1;
   1273 		break;
   1274 	case BUFFEREVENT_SSL_CONNECTING:
   1275 		SSL_set_connect_state(bev_ssl->ssl);
   1276 		if (set_handshake_callbacks(bev_ssl, fd) < 0)
   1277 			return -1;
   1278 		break;
   1279 	case BUFFEREVENT_SSL_OPEN:
   1280 		if (set_open_callbacks(bev_ssl, fd) < 0)
   1281 			return -1;
   1282 		break;
   1283 	default:
   1284 		return -1;
   1285 	}
   1286 
   1287 	return 0;
   1288 }
   1289 
   1290 static int
   1291 be_openssl_ctrl(struct bufferevent *bev,
   1292     enum bufferevent_ctrl_op op, union bufferevent_ctrl_data *data)
   1293 {
   1294 	struct bufferevent_openssl *bev_ssl = upcast(bev);
   1295 	switch (op) {
   1296 	case BEV_CTRL_SET_FD:
   1297 		if (!bev_ssl->underlying) {
   1298 			BIO *bio;
   1299 			bio = BIO_new_socket(data->fd, 0);
   1300 			SSL_set_bio(bev_ssl->ssl, bio, bio);
   1301 		} else {
   1302 			BIO *bio;
   1303 			if (!(bio = BIO_new_bufferevent(bev_ssl->underlying, 0)))
   1304 				return -1;
   1305 			SSL_set_bio(bev_ssl->ssl, bio, bio);
   1306 		}
   1307 
   1308 		return be_openssl_set_fd(bev_ssl, bev_ssl->old_state, data->fd);
   1309 	case BEV_CTRL_GET_FD:
   1310 		if (bev_ssl->underlying) {
   1311 			data->fd = event_get_fd(&bev_ssl->underlying->ev_read);
   1312 		} else {
   1313 			data->fd = event_get_fd(&bev->ev_read);
   1314 		}
   1315 		return 0;
   1316 	case BEV_CTRL_GET_UNDERLYING:
   1317 		data->ptr = bev_ssl->underlying;
   1318 		return 0;
   1319 	case BEV_CTRL_CANCEL_ALL:
   1320 	default:
   1321 		return -1;
   1322 	}
   1323 }
   1324 
   1325 SSL *
   1326 bufferevent_openssl_get_ssl(struct bufferevent *bufev)
   1327 {
   1328 	struct bufferevent_openssl *bev_ssl = upcast(bufev);
   1329 	if (!bev_ssl)
   1330 		return NULL;
   1331 	return bev_ssl->ssl;
   1332 }
   1333 
   1334 static struct bufferevent *
   1335 bufferevent_openssl_new_impl(struct event_base *base,
   1336     struct bufferevent *underlying,
   1337     evutil_socket_t fd,
   1338     SSL *ssl,
   1339     enum bufferevent_ssl_state state,
   1340     int options)
   1341 {
   1342 	struct bufferevent_openssl *bev_ssl = NULL;
   1343 	struct bufferevent_private *bev_p = NULL;
   1344 	int tmp_options = options & ~BEV_OPT_THREADSAFE;
   1345 
   1346 	if (underlying != NULL && fd >= 0)
   1347 		return NULL; /* Only one can be set. */
   1348 
   1349 	if (!(bev_ssl = mm_calloc(1, sizeof(struct bufferevent_openssl))))
   1350 		goto err;
   1351 
   1352 	bev_p = &bev_ssl->bev;
   1353 
   1354 	if (bufferevent_init_common_(bev_p, base,
   1355 		&bufferevent_ops_openssl, tmp_options) < 0)
   1356 		goto err;
   1357 
   1358 	/* Don't explode if we decide to realloc a chunk we're writing from in
   1359 	 * the output buffer. */
   1360 	SSL_set_mode(ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
   1361 
   1362 	bev_ssl->underlying = underlying;
   1363 	bev_ssl->ssl = ssl;
   1364 
   1365 	bev_ssl->outbuf_cb = evbuffer_add_cb(bev_p->bev.output,
   1366 	    be_openssl_outbuf_cb, bev_ssl);
   1367 
   1368 	if (options & BEV_OPT_THREADSAFE)
   1369 		bufferevent_enable_locking_(&bev_ssl->bev.bev, NULL);
   1370 
   1371 	if (underlying) {
   1372 		bufferevent_init_generic_timeout_cbs_(&bev_ssl->bev.bev);
   1373 		bufferevent_incref_(underlying);
   1374 	}
   1375 
   1376 	bev_ssl->old_state = state;
   1377 	bev_ssl->last_write = -1;
   1378 
   1379 	init_bio_counts(bev_ssl);
   1380 
   1381 	fd = be_openssl_auto_fd(bev_ssl, fd);
   1382 	if (be_openssl_set_fd(bev_ssl, state, fd))
   1383 		goto err;
   1384 
   1385 	if (underlying) {
   1386 		bufferevent_setwatermark(underlying, EV_READ, 0, 0);
   1387 		bufferevent_enable(underlying, EV_READ|EV_WRITE);
   1388 		if (state == BUFFEREVENT_SSL_OPEN)
   1389 			bufferevent_suspend_read_(underlying,
   1390 			    BEV_SUSPEND_FILT_READ);
   1391 	}
   1392 
   1393 	return &bev_ssl->bev.bev;
   1394 err:
   1395 	if (bev_ssl)
   1396 		bufferevent_free(&bev_ssl->bev.bev);
   1397 	return NULL;
   1398 }
   1399 
   1400 struct bufferevent *
   1401 bufferevent_openssl_filter_new(struct event_base *base,
   1402     struct bufferevent *underlying,
   1403     SSL *ssl,
   1404     enum bufferevent_ssl_state state,
   1405     int options)
   1406 {
   1407 	/* We don't tell the BIO to close the bufferevent; we do it ourselves
   1408 	 * on be_openssl_destruct */
   1409 	int close_flag = 0; /* options & BEV_OPT_CLOSE_ON_FREE; */
   1410 	BIO *bio;
   1411 	if (!underlying)
   1412 		return NULL;
   1413 	if (!(bio = BIO_new_bufferevent(underlying, close_flag)))
   1414 		return NULL;
   1415 
   1416 	SSL_set_bio(ssl, bio, bio);
   1417 
   1418 	return bufferevent_openssl_new_impl(
   1419 		base, underlying, -1, ssl, state, options);
   1420 }
   1421 
   1422 struct bufferevent *
   1423 bufferevent_openssl_socket_new(struct event_base *base,
   1424     evutil_socket_t fd,
   1425     SSL *ssl,
   1426     enum bufferevent_ssl_state state,
   1427     int options)
   1428 {
   1429 	/* Does the SSL already have an fd? */
   1430 	BIO *bio = SSL_get_wbio(ssl);
   1431 	long have_fd = -1;
   1432 
   1433 	if (bio)
   1434 		have_fd = BIO_get_fd(bio, NULL);
   1435 
   1436 	if (have_fd >= 0) {
   1437 		/* The SSL is already configured with an fd. */
   1438 		if (fd < 0) {
   1439 			/* We should learn the fd from the SSL. */
   1440 			fd = (evutil_socket_t) have_fd;
   1441 		} else if (have_fd == (long)fd) {
   1442 			/* We already know the fd from the SSL; do nothing */
   1443 		} else {
   1444 			/* We specified an fd different from that of the SSL.
   1445 			   This is probably an error on our part.  Fail. */
   1446 			return NULL;
   1447 		}
   1448 		(void) BIO_set_close(bio, 0);
   1449 	} else {
   1450 		/* The SSL isn't configured with a BIO with an fd. */
   1451 		if (fd >= 0) {
   1452 			/* ... and we have an fd we want to use. */
   1453 			bio = BIO_new_socket(fd, 0);
   1454 			SSL_set_bio(ssl, bio, bio);
   1455 		} else {
   1456 			/* Leave the fd unset. */
   1457 		}
   1458 	}
   1459 
   1460 	return bufferevent_openssl_new_impl(
   1461 		base, NULL, fd, ssl, state, options);
   1462 }
   1463 
   1464 int
   1465 bufferevent_openssl_get_allow_dirty_shutdown(struct bufferevent *bev)
   1466 {
   1467 	int allow_dirty_shutdown = -1;
   1468 	struct bufferevent_openssl *bev_ssl;
   1469 	BEV_LOCK(bev);
   1470 	bev_ssl = upcast(bev);
   1471 	if (bev_ssl)
   1472 		allow_dirty_shutdown = bev_ssl->allow_dirty_shutdown;
   1473 	BEV_UNLOCK(bev);
   1474 	return allow_dirty_shutdown;
   1475 }
   1476 
   1477 void
   1478 bufferevent_openssl_set_allow_dirty_shutdown(struct bufferevent *bev,
   1479     int allow_dirty_shutdown)
   1480 {
   1481 	struct bufferevent_openssl *bev_ssl;
   1482 	BEV_LOCK(bev);
   1483 	bev_ssl = upcast(bev);
   1484 	if (bev_ssl)
   1485 		bev_ssl->allow_dirty_shutdown = !!allow_dirty_shutdown;
   1486 	BEV_UNLOCK(bev);
   1487 }
   1488 
   1489 unsigned long
   1490 bufferevent_get_openssl_error(struct bufferevent *bev)
   1491 {
   1492 	unsigned long err = 0;
   1493 	struct bufferevent_openssl *bev_ssl;
   1494 	BEV_LOCK(bev);
   1495 	bev_ssl = upcast(bev);
   1496 	if (bev_ssl && bev_ssl->n_errors) {
   1497 		err = bev_ssl->errors[--bev_ssl->n_errors];
   1498 	}
   1499 	BEV_UNLOCK(bev);
   1500 	return err;
   1501 }
   1502