Home | History | Annotate | Download | only in libvncserver
      1 /*
      2  * rfbssl_openssl.c - Secure socket funtions (openssl version)
      3  */
      4 
      5 /*
      6  *  Copyright (C) 2011 Gernot Tenchio
      7  *
      8  *  This is free software; you can redistribute it and/or modify
      9  *  it under the terms of the GNU General Public License as published by
     10  *  the Free Software Foundation; either version 2 of the License, or
     11  *  (at your option) any later version.
     12  *
     13  *  This software is distributed in the hope that it will be useful,
     14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
     15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     16  *  GNU General Public License for more details.
     17  *
     18  *  You should have received a copy of the GNU General Public License
     19  *  along with this software; if not, write to the Free Software
     20  *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
     21  *  USA.
     22  */
     23 
     24 #include "rfbssl.h"
     25 #include <openssl/ssl.h>
     26 #include <openssl/err.h>
     27 
     28 struct rfbssl_ctx {
     29     SSL_CTX *ssl_ctx;
     30     SSL     *ssl;
     31 };
     32 
     33 static void rfbssl_error(void)
     34 {
     35     char buf[1024];
     36     unsigned long e = ERR_get_error();
     37     rfbErr("%s (%ld)\n", ERR_error_string(e, buf), e);
     38 }
     39 
     40 int rfbssl_init(rfbClientPtr cl)
     41 {
     42     char *keyfile;
     43     int r, ret = -1;
     44     struct rfbssl_ctx *ctx;
     45 
     46     SSL_library_init();
     47     SSL_load_error_strings();
     48 
     49     if (cl->screen->sslkeyfile && *cl->screen->sslkeyfile) {
     50       keyfile = cl->screen->sslkeyfile;
     51     } else {
     52       keyfile = cl->screen->sslcertfile;
     53     }
     54 
     55     if (NULL == (ctx = malloc(sizeof(struct rfbssl_ctx)))) {
     56 	rfbErr("OOM\n");
     57     } else if (!cl->screen->sslcertfile || !cl->screen->sslcertfile[0]) {
     58 	rfbErr("SSL connection but no cert specified\n");
     59     } else if (NULL == (ctx->ssl_ctx = SSL_CTX_new(TLSv1_server_method()))) {
     60 	rfbssl_error();
     61     } else if (SSL_CTX_use_PrivateKey_file(ctx->ssl_ctx, keyfile, SSL_FILETYPE_PEM) <= 0) {
     62 	rfbErr("Unable to load private key file %s\n", keyfile);
     63     } else if (SSL_CTX_use_certificate_file(ctx->ssl_ctx, cl->screen->sslcertfile, SSL_FILETYPE_PEM) <= 0) {
     64 	rfbErr("Unable to load certificate file %s\n", cl->screen->sslcertfile);
     65     } else if (NULL == (ctx->ssl = SSL_new(ctx->ssl_ctx))) {
     66 	rfbErr("SSL_new failed\n");
     67 	rfbssl_error();
     68     } else if (!(SSL_set_fd(ctx->ssl, cl->sock))) {
     69 	rfbErr("SSL_set_fd failed\n");
     70 	rfbssl_error();
     71     } else {
     72 	while ((r = SSL_accept(ctx->ssl)) < 0) {
     73 	    if (SSL_get_error(ctx->ssl, r) != SSL_ERROR_WANT_READ)
     74 		break;
     75 	}
     76 	if (r < 0) {
     77 	    rfbErr("SSL_accept failed %d\n", SSL_get_error(ctx->ssl, r));
     78 	} else {
     79 	    cl->sslctx = (rfbSslCtx *)ctx;
     80 	    ret = 0;
     81 	}
     82     }
     83     return ret;
     84 }
     85 
     86 int rfbssl_write(rfbClientPtr cl, const char *buf, int bufsize)
     87 {
     88     int ret;
     89     struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
     90 
     91     while ((ret = SSL_write(ctx->ssl, buf, bufsize)) <= 0) {
     92 	if (SSL_get_error(ctx->ssl, ret) != SSL_ERROR_WANT_WRITE)
     93 	    break;
     94     }
     95     return ret;
     96 }
     97 
     98 int rfbssl_peek(rfbClientPtr cl, char *buf, int bufsize)
     99 {
    100     int ret;
    101     struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
    102 
    103     while ((ret = SSL_peek(ctx->ssl, buf, bufsize)) <= 0) {
    104 	if (SSL_get_error(ctx->ssl, ret) != SSL_ERROR_WANT_READ)
    105 	    break;
    106     }
    107     return ret;
    108 }
    109 
    110 int rfbssl_read(rfbClientPtr cl, char *buf, int bufsize)
    111 {
    112     int ret;
    113     struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
    114 
    115     while ((ret = SSL_read(ctx->ssl, buf, bufsize)) <= 0) {
    116 	if (SSL_get_error(ctx->ssl, ret) != SSL_ERROR_WANT_READ)
    117 	    break;
    118     }
    119     return ret;
    120 }
    121 
    122 int rfbssl_pending(rfbClientPtr cl)
    123 {
    124     struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
    125     return SSL_pending(ctx->ssl);
    126 }
    127 
    128 void rfbssl_destroy(rfbClientPtr cl)
    129 {
    130     struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
    131     if (ctx->ssl)
    132 	SSL_free(ctx->ssl);
    133     if (ctx->ssl_ctx)
    134 	SSL_CTX_free(ctx->ssl_ctx);
    135 }
    136