DaKaZ DaKaZ - 3 months ago 41
C Question

SSL_accept with edge triggered non-blocking epoll always returns SSL_ERROR_WANT_READ

There are 3 of us working on adding SSL to epoll based server but after 3 days of pull-your-hair out frustration we decided to ask the world for help.

The problem is that

SSL_accept()
always returns
SSL_ERROR_WANT_READ
, we DO have epoll events for EPOLLIN and EPOLLOUT for the associated socket but even when we get an event and recall the
SSL_accept
it just returns the want read.... so frustrating.

There seems to be no good working example of using openssl with epoll that we can find, PLEASE send us to code samples if there are any, we have spent days on the this problem and read about every SE/SO message and spent hours on the openssl mailing list.

We have a basic app structure that looks like this for the accept:

if (http_sock == source_fd || https_sock == source_fd) {
// new connection
req_type_t req_type = (http_sock == source_fd) ? HTTP : HTTPS;
int infd;

while (1) {
struct sockaddr_in in_addr;
socklen_t in_addr_len;
ZLOG_DBG(zlog_cat, "New %s request on source_fd: %d http_sock: %d https_sock: %d", req_type == HTTP ? "HTTP" : "HTTPS", source_fd, http_sock, https_sock);
in_addr_len = sizeof in_addr;
infd = accept4(source_fd, &in_addr, &in_addr_len, SOCK_NONBLOCK);
if (infd <= 0)
{
if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) {
/* We have processed all incoming
connections. */
break;
} else {
ZLOG_ERR(zlog_cat, "ERROR: accept");
break;
}
}

req = new_req_data(kds, req_type, &g_data, infd, &in_addr, in_addr_len, cert_q);


event.events = EPOLLIN | EPOLLRDHUP | EPOLLET;

if (req->type == HTTPS) {
req->c_tls.want_ssl_accept = 1;
event.events = EPOLLIN | EPOLLRDHUP | EPOLLOUT | EPOLLET;
ssl_establish(req);
}
if ( (s = set_socket_blocking(req->client.fd, 1)) < 0)
err(EXIT_FAILURE, "[%s:%d] set_socket_blocking()", __FUNCTION__, __LINE__);

epoll_event_t *epoll_event = new_epoll_event_t(infd, req);

event.data.ptr = epoll_event;
// add new infd to our epoll listener
if ( (s = epoll_ctl(efd, EPOLL_CTL_ADD, req->client.fd, &event)) == -1 )
err(EXIT_FAILURE, "epoll_ctl(http) %s", strerror(errno));
}
ZLOG_DBG(zlog_cat, "New %s connection from %s:%d on FD %d mac_address: %s", req->type ? "HTTPS" : "HTTP",
inet_ntoa(req->client.sock.sin_addr), ntohs(req->client.sock.sin_port), req->client.fd, req->device->mac_address);
g_data.conn_counter++;
continue; // continue to next accept event

} // end new connection


The
ssl_establish()
method looks like this:

int ssl_establish(req_data_t *req) {
unsigned long e;
int ssl_err, rc;

// create openssl server context and ssl object
if ((req->s_tls.ctx = create_context(0)) == NULL) {
ZLOG_ERR(zlog_cat, "create_context() %s", strerror(errno));
return -1;
}

if ((req->s_tls.ssl = SSL_new(req->s_tls.ctx)) == NULL) {
ZLOG_ERR(zlog_cat, "SSL_new() %s", strerror(errno));
return -1;
}

// Establishing SSL/TLS connections with client
if (req->https_def_rsa != NULL && req->https_def_cert != NULL) {
req->c_tls.servername = req->https_def_domain;
req->c_tls.cert = req->https_def_cert;
req->c_tls.rsa_key = req->https_def_rsa;
req->c_tls.dest_ip = req->orig.sock.sin_addr.s_addr;
req->c_tls.dc_cache = req->dc_cache;
req->c_tls.q_entry = NULL;

if (req->c_tls.ctx == NULL ) {
if ((req->c_tls.ctx = create_context(1)) == NULL)
ZLOG_ERR(zlog_cat, "ERROR create_context");
if (configure_context(&req->c_tls))
ZLOG_ERR(zlog_cat, "ERROR configure_context");
if ((req->c_tls.ssl = SSL_new(req->c_tls.ctx)) == NULL)
ZLOG_ERR(zlog_cat, "ERROR ssl_new");

// SSL_set_mode(req->c_tls.ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); // This doesn't seem to do anything

if (SSL_set_fd(req->c_tls.ssl, req->client.fd) == 0)
ZLOG_ERR(zlog_cat, "ERROR ssl_set_fd");
}
req->client.ssl = req->c_tls.ssl;
}
if ((rc = SSL_accept(req->c_tls.ssl)) < 0) {
ssl_err = SSL_get_error(req->c_tls.ssl, rc);

switch (ssl_err) {
case SSL_ERROR_WANT_READ:
ZLOG_DBG(zlog_cat, "SSL_accept SSL_ERROR_WANT_READ");
req->c_tls.want_ssl_accept = 1;
return 0;
break;
case SSL_ERROR_WANT_WRITE:
ZLOG_DBG(zlog_cat, "SSL_accept SSL_ERROR_WANT_WRITE");
req->c_tls.want_ssl_accept = 1;
return 0;
break;
case SSL_ERROR_ZERO_RETURN:
ZLOG_ERR(zlog_cat, "SSL_accept SSL_ERROR_ZERO_RETURN");
req->c_tls.want_ssl_accept = 0;
return -1;
break;
case SSL_ERROR_WANT_X509_LOOKUP:
ZLOG_ERR(zlog_cat, "SSL_accept SSL_ERROR_WANT_X509_LOOKUP");
req->c_tls.want_ssl_accept = 0;
return -1;
break;
case SSL_ERROR_SYSCALL:
ZLOG_ERR(zlog_cat, "SSL_accept SSL_ERROR_SYSCALL");
req->c_tls.want_ssl_accept = 0;
return -1;
break;
case SSL_ERROR_SSL:
ZLOG_ERR(zlog_cat, "SSL_accept SSL_ERROR_SSL");
req->c_tls.want_ssl_accept = 0;
return -1;
break;
}
}
ZLOG_DBG(zlog_cat, "SSL_accept success");
req->c_tls.want_ssl_accept = 0;
return 1;
}


In the pollin / pollout handlers we do this:

if (req->type == HTTPS && req->c_tls.want_ssl_accept) {
ZLOG_DBG(zlog_cat, "ssl_establish EPOLLIN");
if ((rc = ssl_establish(req)) <= 0) {
ZLOG_DBG(zlog_cat, "ssl_establish EPOLLIN retry...");
continue;
}
}


Our log looks like this:

2016-08-03 23:26:32.224 New HTTPS connection from 192.168.1.195:52659 on FD 15 mac_address: e4:f8:9c:85:63:99
2016-08-03 23:26:32.224 epoll_wait returned 1
2016-08-03 23:26:32.224 epoll event number: 0 on fd: 15
2016-08-03 23:26:32.224 Reading data from Client, source_fd: 15 source_sock->fd: 15 dest_sock->fd: -1 mac_address: e4:f8:9c:85:63:99
2016-08-03 23:26:32.224 ssl_establish EPOLLIN
2016-08-03 23:26:32.484 SSL_accept SSL_ERROR_WANT_READ
2016-08-03 23:26:32.484 ssl_establish EPOLLIN retry...
2016-08-03 23:26:32.954 New HTTPS connection from 192.168.1.195:52660 on FD 16 mac_address: e4:f8:9c:85:63:99
2016-08-03 23:26:32.954 epoll_wait returned 1
2016-08-03 23:26:32.954 epoll event number: 0 on fd: 16
2016-08-03 23:26:32.954 Establishing SSL (EPOLLOUT)
2016-08-03 23:26:32.974 SSL_accept SSL_ERROR_WANT_READ
2016-08-03 23:26:32.974 ssl_establish EPOLLOUT retry...

Answer

Okay - I finally got it working. If anyone stumbles across this, don't use SSL_accept().

I added some state tracking to my main struct and the updated the ssl_establish() method to look like this:

int ssl_establish(req_data_t *req, epoll_event_t *epoll_event ) {
    unsigned long e;
    int ssl_err, rc;
    struct epoll_event event;
    memset(&event, 0, sizeof(struct epoll_event));
    event.events = req->client.events;
    event.data.ptr = epoll_event;

    if (!req->client.tcp_connected) {
        struct pollfd pfd;
        pfd.fd = req->client.fd;
        pfd.events = POLLOUT | POLLERR;
        int r = poll(&pfd, 1, 0);
        if (r == 1 && pfd.revents == POLLOUT) {
            ZLOG_DBG(zlog_cat, "tcp connected fd %d", req->client.fd);
            req->client.tcp_connected = 1;
            req->client.events = EPOLLIN | EPOLLOUT | EPOLLRDHUP | EPOLLET;
            event.events = req->client.events;
            if(epoll_ctl(req->efd, EPOLL_CTL_MOD, req->client.fd, &event) != 0)
               ZLOG_ERR(zlog_cat, "ERROR: unable to modify epoll_ctl");
        } else {
            ZLOG_ERR(zlog_cat, "[%d | %d] ERROR poll fd return %d revents %d", req->client.fd, req->state, r, pfd.revents);
            return -1;
        }
    }


    // create openssl server context and ssl object
    if (!req->s_tls.ctx)
        if ((req->s_tls.ctx = create_context(0)) == NULL) {
            ZLOG_ERR(zlog_cat, "create_context() %s", strerror(errno));
            return -1;
        }

    if (!req->s_tls.ssl)
        if ((req->s_tls.ssl = SSL_new(req->s_tls.ctx)) == NULL) {
            ZLOG_ERR(zlog_cat, "SSL_new() %s", strerror(errno));
            return -1;
        }

    req->orig.ssl = req->s_tls.ssl;
    req->proxy.ssl = req->s_tls.ssl;

    // Establishing SSL/TLS connections with client
    if (req->https_def_rsa != NULL && req->https_def_cert != NULL) {
        req->c_tls.servername = req->https_def_domain;
        req->c_tls.cert = req->https_def_cert;
        req->c_tls.rsa_key = req->https_def_rsa;
        req->c_tls.dest_ip = req->orig.sock.sin_addr.s_addr;
        req->c_tls.kudoso = req->kudoso;
        req->c_tls.dc_cache = req->dc_cache;
        req->c_tls.q_entry = NULL;

        if (!req->errBio)
            req->errBio = BIO_new_fd(2, BIO_NOCLOSE);


        if (!req->c_tls.ctx) {
            if ((req->c_tls.ctx = create_context(1)) == NULL)
                ZLOG_ERR(zlog_cat, "ERROR create_context");
            if (configure_context(&req->c_tls))
                ZLOG_ERR(zlog_cat, "ERROR configure_context");
        }
        if (!req->c_tls.ssl) {
            if ((req->c_tls.ssl = SSL_new(req->c_tls.ctx)) == NULL)
                ZLOG_ERR(zlog_cat, "ERROR ssl_new");

            // SSL_set_mode(req->c_tls.ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);

            if (SSL_set_fd(req->c_tls.ssl, req->client.fd) == 0)
                ZLOG_ERR(zlog_cat, "ERROR ssl_set_fd");

            SSL_set_accept_state(req->c_tls.ssl);
            req->client.ssl = req->c_tls.ssl;
        }
    }
    int r = SSL_do_handshake(req->c_tls.ssl);
    if (r == 1) {
        req->client.ssl_connected = 1;
        ZLOG_DBG(zlog_cat, "[%d | %d] ssl connected", req->client.fd, req->state);
        req->client.events = EPOLLIN | EPOLLRDHUP | EPOLLET;
        event.events = req->client.events;
        if(epoll_ctl(req->efd, EPOLL_CTL_MOD, req->client.fd, &event) != 0)
           ZLOG_ERR(zlog_cat, "ERROR: unable to modify epoll_ctl");
        return 1;
    }
    int err = SSL_get_error(req->c_tls.ssl, r);
    int oldev = req->client.events;
    if (err == SSL_ERROR_WANT_WRITE) {
        req->client.events |= EPOLLOUT;
        req->client.events &= ~EPOLLIN;
        ZLOG_DBG(zlog_cat, "do_handshake return want write set events %d", req->client.events);
        if (oldev == req->client.events) return 0;
    } else if (err == SSL_ERROR_WANT_READ) {
        req->client.events |= EPOLLIN;
        req->client.events &= ~EPOLLOUT;
        ZLOG_DBG(zlog_cat, "do_handshake return want read set events %d", req->client.events);
        if (oldev == req->client.events) return 0;
    } else {
        ZLOG_ERR(zlog_cat, "ERROR SSL_do_handshake return %d error %d errno %d msg %s", r, err, errno, strerror(errno));
        ERR_print_errors(req->errBio);
        return -1;
    }
    event.events = req->client.events;
    if(epoll_ctl(req->efd, EPOLL_CTL_MOD, req->client.fd, &event) != 0)
       ZLOG_ERR(zlog_cat, "[%d | %d] ERROR: unable to modify epoll_ctl", req->client.fd, req->state);
    return 0;
}