diff --git a/include/libwebsockets/lws-context-vhost.h b/include/libwebsockets/lws-context-vhost.h index 522a8fad3..b3de140b5 100644 --- a/include/libwebsockets/lws-context-vhost.h +++ b/include/libwebsockets/lws-context-vhost.h @@ -453,9 +453,8 @@ struct lws_context_creation_info { int simultaneous_ssl_restriction; /**< CONTEXT: 0 (no limit) or limit of simultaneous SSL sessions * possible.*/ - int ssl_handshake_serialize; - /**< CONTEXT: 0 disables ssl handshake serialization (default). - * 1 enables ssl handshake serialization. */ + int simultaneous_ssl_handshake_restriction; + /**< CONTEXT: 0 (no limit) or limit of simultaneous SSL handshakes ongoing */ int ssl_info_event_mask; /**< VHOST: mask of ssl events to be reported on LWS_CALLBACK_SSL_INFO * callback for connections on this vhost. The mask values are of diff --git a/lib/core-net/client/connect.c b/lib/core-net/client/connect.c index 4b45bbe07..1844abef8 100644 --- a/lib/core-net/client/connect.c +++ b/lib/core-net/client/connect.c @@ -530,8 +530,8 @@ bail3: bail: #if defined(LWS_WITH_TLS) - if (wsi->tls.ssl && wsi->tls_borrowed) - lws_tls_restrict_return(i->context); + if (wsi->tls.ssl) + lws_tls_restrict_return(wsi); #endif lws_free_set_NULL(wsi->stash); diff --git a/lib/core-net/private-lib-core-net.h b/lib/core-net/private-lib-core-net.h index 1f566178b..3f5073110 100644 --- a/lib/core-net/private-lib-core-net.h +++ b/lib/core-net/private-lib-core-net.h @@ -809,6 +809,7 @@ struct lws { unsigned int client_proxy_onward:1; #endif unsigned int tls_borrowed:1; + unsigned int tls_borrowed_hs:1; unsigned int tls_read_wanted_write:1; #ifdef LWS_WITH_ACCESS_LOG diff --git a/lib/core/context.c b/lib/core/context.c index 95219e473..d55ee5156 100644 --- a/lib/core/context.c +++ b/lib/core/context.c @@ -915,7 +915,8 @@ lws_create_context(const struct lws_context_creation_info *info) #if defined(LWS_WITH_TLS) && defined(LWS_WITH_NETWORK) context->simultaneous_ssl_restriction = info->simultaneous_ssl_restriction; - context->ssl_handshake_serialize = info->ssl_handshake_serialize; + context->simultaneous_ssl_handshake_restriction = + info->simultaneous_ssl_handshake_restriction; #endif context->options = info->options; diff --git a/lib/core/private-lib-core.h b/lib/core/private-lib-core.h index 51078a5ab..8ba08520d 100644 --- a/lib/core/private-lib-core.h +++ b/lib/core/private-lib-core.h @@ -702,7 +702,8 @@ struct lws_context { unsigned int max_http_header_pool; int simultaneous_ssl_restriction; int simultaneous_ssl; - int ssl_handshake_serialize; + int simultaneous_ssl_handshake_restriction; + int simultaneous_ssl_handshake; #if defined(LWS_WITH_TLS_JIT_TRUST) int vh_idle_grace_ms; #endif @@ -721,6 +722,8 @@ struct lws_context { lws_route_uidx_t route_uidx; #endif + char tls_gate_accepts; + unsigned int deprecated:1; unsigned int inside_context_destroy:1; unsigned int being_destroyed:1; diff --git a/lib/tls/mbedtls/mbedtls-ssl.c b/lib/tls/mbedtls/mbedtls-ssl.c index 436f34aa0..5fe92205a 100644 --- a/lib/tls/mbedtls/mbedtls-ssl.c +++ b/lib/tls/mbedtls/mbedtls-ssl.c @@ -278,8 +278,7 @@ lws_ssl_close(struct lws *wsi) SSL_free(wsi->tls.ssl); wsi->tls.ssl = NULL; - if (wsi->tls_borrowed) - lws_tls_restrict_return(wsi->a.context); + lws_tls_restrict_return(wsi); return 1; /* handled */ } diff --git a/lib/tls/openssl/openssl-ssl.c b/lib/tls/openssl/openssl-ssl.c index 09844a42a..cf4d2b8c6 100644 --- a/lib/tls/openssl/openssl-ssl.c +++ b/lib/tls/openssl/openssl-ssl.c @@ -469,8 +469,7 @@ lws_ssl_close(struct lws *wsi) SSL_free(wsi->tls.ssl); wsi->tls.ssl = NULL; - if (wsi->tls_borrowed) - lws_tls_restrict_return(wsi->a.context); + lws_tls_restrict_return(wsi); // lwsl_notice("%s: ssl restr %d, simul %d\n", __func__, // wsi->a.context->simultaneous_ssl_restriction, diff --git a/lib/tls/private-lib-tls.h b/lib/tls/private-lib-tls.h index 23fc2a727..13ac69266 100644 --- a/lib/tls/private-lib-tls.h +++ b/lib/tls/private-lib-tls.h @@ -126,10 +126,13 @@ enum lws_tls_extant { #endif int -lws_tls_restrict_borrow(struct lws_context *context); +lws_tls_restrict_borrow(struct lws *wsi); void -lws_tls_restrict_return(struct lws_context *context); +lws_tls_restrict_return(struct lws *wsi); + +void +lws_tls_restrict_return_handshake(struct lws *wsi); typedef SSL lws_tls_conn; typedef SSL_CTX lws_tls_ctx; diff --git a/lib/tls/tls-client.c b/lib/tls/tls-client.c index 6009e4352..9d93a3dae 100644 --- a/lib/tls/tls-client.c +++ b/lib/tls/tls-client.c @@ -24,43 +24,6 @@ #include "private-lib-core.h" -static int -lws_ssl_handshake_serialize(struct lws_context *ctx, struct lws *wsi) -{ - struct lws_vhost *vh = ctx->vhost_list; -#if LWS_MAX_SMP > 1 - int tsi = lws_pthread_self_to_tsi(ctx); -#else - int tsi = 0; -#endif - struct lws_context_per_thread *pt = &ctx->pt[tsi]; - unsigned int n; - - while (vh) { - for (n = 0; n < pt->fds_count; n++) { - struct lws *w = wsi_from_fd(ctx, pt->fds[n].fd); - - if (!w || w->tsi != tsi || w->a.vhost != vh || wsi == w) - continue; - - /* Now we found other vhost's wsi in process */ - if (lwsi_role_mqtt(w)) { - /* MQTT TLS connection not established yet. - * Let it finish. - */ - if (lwsi_state(w) != LRS_ESTABLISHED) - return 1; - } else { - /* H1/H2 not finished yet. Let it finish. */ - if (lwsi_state(w) != LRS_DEAD_SOCKET) - return 1; - } - } - vh = vh->vhost_next; - } - return 0; -} - static int lws_ssl_client_connect1(struct lws *wsi, char *errbuf, size_t len) { @@ -69,8 +32,10 @@ lws_ssl_client_connect1(struct lws *wsi, char *errbuf, size_t len) n = lws_tls_client_connect(wsi, errbuf, len); switch (n) { case LWS_SSL_CAPABLE_ERROR: + lws_tls_restrict_return_handshake(wsi); return -1; case LWS_SSL_CAPABLE_DONE: + lws_tls_restrict_return_handshake(wsi); lws_metrics_caliper_report(wsi->cal_conn, METRES_GO); #if defined(LWS_WITH_CONMON) wsi->conmon.ciu_tls = (lws_conmon_interval_us_t) @@ -100,6 +65,7 @@ lws_ssl_client_connect2(struct lws *wsi, char *errbuf, size_t len) switch (n) { case LWS_SSL_CAPABLE_ERROR: + lws_tls_restrict_return_handshake(wsi); // lws_snprintf(errbuf, len, "client connect failed"); return -1; case LWS_SSL_CAPABLE_DONE: @@ -115,6 +81,8 @@ lws_ssl_client_connect2(struct lws *wsi, char *errbuf, size_t len) } } + lws_tls_restrict_return_handshake(wsi); + if (lws_tls_client_confirm_peer_cert(wsi, errbuf, len)) { lws_metrics_caliper_report(wsi->cal_conn, METRES_NOGO); return -1; @@ -221,23 +189,12 @@ lws_client_create_tls(struct lws *wsi, const char **pcce, int do_c1) if (!wsi->tls.ssl) { #if defined(LWS_WITH_TLS) - if (!wsi->transaction_from_pipeline_queue) { - if (lws_tls_restrict_borrow(wsi->a.context)) { - *pcce = "tls restriction limit"; - return CCTLS_RETURN_ERROR; - } - wsi->tls_borrowed = 1; - if (wsi->a.context->ssl_handshake_serialize) { - if (lws_ssl_handshake_serialize(wsi->a.context, wsi)) { - lws_tls_restrict_return(wsi->a.context); - wsi->tls_borrowed = 0; - *pcce = "ssl handshake serialization"; - return CCTLS_RETURN_ERROR; - } - } + if (!wsi->transaction_from_pipeline_queue && + lws_tls_restrict_borrow(wsi)) { + *pcce = "tls restriction limit"; + return CCTLS_RETURN_ERROR; } #endif - if (lws_ssl_client_bio_create(wsi) < 0) { *pcce = "bio_create failed"; return CCTLS_RETURN_ERROR; diff --git a/lib/tls/tls-network.c b/lib/tls/tls-network.c index 0698e1ab5..b2038d537 100644 --- a/lib/tls/tls-network.c +++ b/lib/tls/tls-network.c @@ -203,6 +203,11 @@ lws_gate_accepts(struct lws_context *context, int on) lwsl_notice("%s: on = %d\n", __func__, on); + if (context->tls_gate_accepts == (char)on) + return 0; + + context->tls_gate_accepts = (char)on; + while (v) { lws_start_foreach_dll(struct lws_dll2 *, d, lws_dll2_get_head(&v->listen_wsi)) { @@ -210,8 +215,8 @@ lws_gate_accepts(struct lws_context *context, int on) listen_list); if (v->tls.use_ssl && - lws_change_pollfd(wsi, on ? 0 : LWS_POLLIN, - on ? LWS_POLLIN : 0)) + lws_change_pollfd(wsi, on ? LWS_POLLIN : 0, + on ? 0 : LWS_POLLIN)) lwsl_notice("%s: Unable to set POLLIN %d\n", __func__, on); } lws_end_foreach_dll(d); diff --git a/lib/tls/tls-server.c b/lib/tls/tls-server.c index 39578c1b8..dbb517834 100644 --- a/lib/tls/tls-server.c +++ b/lib/tls/tls-server.c @@ -144,18 +144,16 @@ lws_server_socket_service_ssl(struct lws *wsi, lws_sockfd_type accept_fd, char f if (accept_fd == LWS_SOCK_INVALID) assert(0); - if (lws_tls_restrict_borrow(context)) { + if (lws_tls_restrict_borrow(wsi)) { lwsl_err("%s: failed on ssl restriction\n", __func__); return 1; } - wsi->tls_borrowed = 1; if (lws_tls_server_new_nonblocking(wsi, accept_fd)) { lwsl_err("%s: failed on lws_tls_server_new_nonblocking\n", __func__); if (accept_fd != LWS_SOCK_INVALID) compatible_close(accept_fd); - if (wsi->tls_borrowed) - lws_tls_restrict_return(context); + lws_tls_restrict_return(wsi); goto fail; } @@ -322,8 +320,10 @@ punt: lwsl_info("SSL_accept says %d\n", n); switch (n) { case LWS_SSL_CAPABLE_DONE: + lws_tls_restrict_return_handshake(wsi); break; case LWS_SSL_CAPABLE_ERROR: + lws_tls_restrict_return_handshake(wsi); lwsl_info("%s: SSL_accept failed socket %u: %d\n", __func__, wsi->desc.sockfd, n); wsi->socket_is_permanently_unusable = 1; diff --git a/lib/tls/tls.c b/lib/tls/tls.c index 9e9a32ca4..6afc76400 100644 --- a/lib/tls/tls.c +++ b/lib/tls/tls.c @@ -46,52 +46,110 @@ alpn_cb(SSL *s, const unsigned char **out, unsigned char *outlen, #endif int -lws_tls_restrict_borrow(struct lws_context *context) +lws_tls_restrict_borrow(struct lws *wsi) { - if (!context->simultaneous_ssl_restriction) - return 0; + struct lws_context *cx = wsi->a.context; - if (context->simultaneous_ssl >= context->simultaneous_ssl_restriction) { + if (cx->simultaneous_ssl_restriction && + cx->simultaneous_ssl >= cx->simultaneous_ssl_restriction) { lwsl_notice("%s: tls connection limit %d\n", __func__, - context->simultaneous_ssl); + cx->simultaneous_ssl); return 1; } - context->simultaneous_ssl++; + if (cx->simultaneous_ssl_handshake_restriction && + cx->simultaneous_ssl_handshake >= + cx->simultaneous_ssl_handshake_restriction) { + lwsl_notice("%s: tls handshake limit %d\n", __func__, + cx->simultaneous_ssl); + return 1; + } + + cx->simultaneous_ssl++; + cx->simultaneous_ssl_handshake++; + wsi->tls_borrowed_hs = 1; + wsi->tls_borrowed = 1; lwsl_info("%s: %d -> %d\n", __func__, - context->simultaneous_ssl - 1, - context->simultaneous_ssl); + cx->simultaneous_ssl - 1, + cx->simultaneous_ssl); - assert(context->simultaneous_ssl <= - context->simultaneous_ssl_restriction); + assert(!cx->simultaneous_ssl_restriction || + cx->simultaneous_ssl <= + cx->simultaneous_ssl_restriction); + assert(!cx->simultaneous_ssl_handshake_restriction || + cx->simultaneous_ssl_handshake <= + cx->simultaneous_ssl_handshake_restriction); #if defined(LWS_WITH_SERVER) - if (context->simultaneous_ssl == context->simultaneous_ssl_restriction) - /* that was the last allowed SSL connection */ - lws_gate_accepts(context, 0); + lws_gate_accepts(cx, + (cx->simultaneous_ssl_restriction && + cx->simultaneous_ssl == cx->simultaneous_ssl_restriction) || + (cx->simultaneous_ssl_handshake_restriction && + cx->simultaneous_ssl_handshake == cx->simultaneous_ssl_handshake_restriction)); #endif return 0; } -void -lws_tls_restrict_return(struct lws_context *context) +static void +_lws_tls_restrict_return(struct lws *wsi) { - if (context->simultaneous_ssl_restriction) { - int n = context->simultaneous_ssl--; + struct lws_context *cx = wsi->a.context; - lwsl_info("%s: %d -> %d\n", __func__, n, - context->simultaneous_ssl); - - assert(context->simultaneous_ssl >= 0); + assert(cx->simultaneous_ssl_handshake >= 0); + assert(cx->simultaneous_ssl >= 0); #if defined(LWS_WITH_SERVER) - if (n == context->simultaneous_ssl_restriction) - /* we made space and can do an accept */ - lws_gate_accepts(context, 1); + lws_gate_accepts(cx, + (cx->simultaneous_ssl_restriction && + cx->simultaneous_ssl == cx->simultaneous_ssl_restriction) || + (cx->simultaneous_ssl_handshake_restriction && + cx->simultaneous_ssl_handshake == cx->simultaneous_ssl_handshake_restriction)); #endif - } +} + +void +lws_tls_restrict_return_handshake(struct lws *wsi) +{ + struct lws_context *cx = wsi->a.context; + + /* we're just returning the hs part */ + + if (!wsi->tls_borrowed_hs) + return; + + wsi->tls_borrowed_hs = 0; /* return it one time per wsi */ + cx->simultaneous_ssl_handshake--; + + lwsl_info("%s: %d -> %d\n", __func__, + cx->simultaneous_ssl_handshake + 1, + cx->simultaneous_ssl_handshake); + + _lws_tls_restrict_return(wsi); +} + +void +lws_tls_restrict_return(struct lws *wsi) +{ + struct lws_context *cx = wsi->a.context; + + if (!wsi->tls_borrowed) + return; + + wsi->tls_borrowed = 0; + cx->simultaneous_ssl--; + + lwsl_info("%s: %d -> %d\n", __func__, + cx->simultaneous_ssl + 1, + cx->simultaneous_ssl); + + /* We're returning everything, even if hs didn't complete */ + + if (wsi->tls_borrowed_hs) + lws_tls_restrict_return_handshake(wsi); + else + _lws_tls_restrict_return(wsi); } void diff --git a/minimal-examples/http-client/minimal-http-client-multi/minimal-http-client-multi.c b/minimal-examples/http-client/minimal-http-client-multi/minimal-http-client-multi.c index fb7c65b9a..d0d11f3fe 100644 --- a/minimal-examples/http-client/minimal-http-client-multi/minimal-http-client-multi.c +++ b/minimal-examples/http-client/minimal-http-client-multi/minimal-http-client-multi.c @@ -586,6 +586,12 @@ int main(int argc, const char **argv) if ((p = lws_cmdline_option(argc, argv, "--limit"))) info.simultaneous_ssl_restriction = atoi(p); + if ((p = lws_cmdline_option(argc, argv, "--ssl-handshake-serialize"))) + /* We only consider simultaneous_ssl_restriction > 1 use cases. + * If ssl isn't limited or only 1 is allowed, we don't care. + */ + info.simultaneous_ssl_handshake_restriction = atoi(p); + context = lws_create_context(&info); if (!context) { lwsl_err("lws init failed\n");