diff --git a/lib/core-net/client/connect.c b/lib/core-net/client/connect.c
index e83dcfbd7..4b45bbe07 100644
--- a/lib/core-net/client/connect.c
+++ b/lib/core-net/client/connect.c
@@ -529,6 +529,11 @@ bail3:
 #endif
 
 bail:
+#if defined(LWS_WITH_TLS)
+	if (wsi->tls.ssl && wsi->tls_borrowed)
+		lws_tls_restrict_return(i->context);
+#endif
+
 	lws_free_set_NULL(wsi->stash);
 	lws_fi_destroy(&wsi->fic);
 	lws_free(wsi);
@@ -536,11 +541,6 @@ bail:
 bail2:
 #endif
 
-#if defined(LWS_WITH_TLS)
-	if (wsi->tls.ssl)
-		lws_tls_restrict_return(i->context);
-#endif
-
 	if (i->pwsi)
 		*i->pwsi = NULL;
 
diff --git a/lib/core-net/private-lib-core-net.h b/lib/core-net/private-lib-core-net.h
index 8bb674900..a3b667d0f 100644
--- a/lib/core-net/private-lib-core-net.h
+++ b/lib/core-net/private-lib-core-net.h
@@ -824,6 +824,7 @@ struct lws {
 	unsigned int			client_bound_sspc:1;
 	unsigned int			client_proxy_onward:1;
 #endif
+	unsigned int                    tls_borrowed:1;
 
 #ifdef LWS_WITH_ACCESS_LOG
 	unsigned int			access_log_pending:1;
diff --git a/lib/tls/mbedtls/mbedtls-ssl.c b/lib/tls/mbedtls/mbedtls-ssl.c
index 771a2f765..e2ad69b56 100644
--- a/lib/tls/mbedtls/mbedtls-ssl.c
+++ b/lib/tls/mbedtls/mbedtls-ssl.c
@@ -276,7 +276,8 @@ lws_ssl_close(struct lws *wsi)
 	SSL_free(wsi->tls.ssl);
 	wsi->tls.ssl = NULL;
 
-	lws_tls_restrict_return(wsi->a.context);
+	if (wsi->tls_borrowed)
+		lws_tls_restrict_return(wsi->a.context);
 
 	return 1; /* handled */
 }
diff --git a/lib/tls/openssl/openssl-ssl.c b/lib/tls/openssl/openssl-ssl.c
index 60a7b05f0..5991a09e4 100644
--- a/lib/tls/openssl/openssl-ssl.c
+++ b/lib/tls/openssl/openssl-ssl.c
@@ -465,7 +465,8 @@ lws_ssl_close(struct lws *wsi)
 	SSL_free(wsi->tls.ssl);
 	wsi->tls.ssl = NULL;
 
-	lws_tls_restrict_return(wsi->a.context);
+	if (wsi->tls_borrowed)
+		lws_tls_restrict_return(wsi->a.context);
 
 	// lwsl_notice("%s: ssl restr %d, simul %d\n", __func__,
 	//		wsi->a.context->simultaneous_ssl_restriction,
diff --git a/lib/tls/tls-client.c b/lib/tls/tls-client.c
index 972438f39..9e8443c4c 100644
--- a/lib/tls/tls-client.c
+++ b/lib/tls/tls-client.c
@@ -184,10 +184,12 @@ lws_client_create_tls(struct lws *wsi, const char **pcce, int do_c1)
 		if (!wsi->tls.ssl) {
 
 #if defined(LWS_WITH_TLS)
-			if (!wsi->transaction_from_pipeline_queue &&
-			    lws_tls_restrict_borrow(wsi->a.context)) {
-				*pcce = "tls restriction limit";
-				return CCTLS_RETURN_ERROR;
+			if (!wsi->transaction_from_pipeline_queue) {
+				if (lws_tls_restrict_borrow(wsi->a.context)) {
+					*pcce = "tls restriction limit";
+					return CCTLS_RETURN_ERROR;
+				}
+				wsi->tls_borrowed = 1;
 			}
 #endif
 
diff --git a/lib/tls/tls-server.c b/lib/tls/tls-server.c
index 887159fa2..39578c1b8 100644
--- a/lib/tls/tls-server.c
+++ b/lib/tls/tls-server.c
@@ -148,12 +148,14 @@ lws_server_socket_service_ssl(struct lws *wsi, lws_sockfd_type accept_fd, char f
 			lwsl_err("%s: failed on ssl restriction\n", __func__);
 			return 1;
 		}
+		wsi->tls_borrowed = 1;
 
 		if (lws_tls_server_new_nonblocking(wsi, accept_fd)) {
 			lwsl_err("%s: failed on lws_tls_server_new_nonblocking\n", __func__);
 			if (accept_fd != LWS_SOCK_INVALID)
 				compatible_close(accept_fd);
-			lws_tls_restrict_return(context);
+			if (wsi->tls_borrowed)
+				lws_tls_restrict_return(context);
 			goto fail;
 		}