diff --git a/lib/ssl.c b/lib/ssl.c
index b033dbf3..1eabad9e 100644
--- a/lib/ssl.c
+++ b/lib/ssl.c
@@ -615,8 +615,8 @@ lws_server_socket_service_ssl(struct libwebsocket_context *context,
 		 * it disabled unless you know it's not a problem for you
 		 */
 
-		if (context->allow_non_ssl_on_ssl_port && n >= 1 &&
-					context->service_buffer[0] >= ' ') {
+		if (context->allow_non_ssl_on_ssl_port) {
+			if (n >= 1 && context->service_buffer[0] >= ' ') {
 			/*
 			 * TLS content-type for Handshake is 0x16
 			 * TLS content-type for ChangeCipherSpec Record is 0x14
@@ -626,11 +626,22 @@ lws_server_socket_service_ssl(struct libwebsocket_context *context,
 			 * kill the SSL for this connection and try to handle
 			 * as a HTTP connection upgrade directly.
 			 */
-			wsi->use_ssl = 0;
-			SSL_shutdown(wsi->ssl);
-			SSL_free(wsi->ssl);
-			wsi->ssl = NULL;
-			goto accepted;
+				wsi->use_ssl = 0;
+				SSL_shutdown(wsi->ssl);
+				SSL_free(wsi->ssl);
+				wsi->ssl = NULL;
+				goto accepted;
+			}
+			if (n == 0 || n == LWS_EAGAIN || n == LWS_EWOULDBLOCK) {
+				/*
+				 * well, we get no way to know ssl or not
+				 * so go around again waiting for something
+				 * to come and give us a hint, or timeout the
+				 * connection.
+				 */
+				m = SSL_ERROR_WANT_READ;
+				goto go_again;
+			}
 		}
 
 		/* normal SSL connection processing path */
@@ -645,7 +656,7 @@ lws_server_socket_service_ssl(struct libwebsocket_context *context,
 		m = SSL_get_error(wsi->ssl, n);
 		lwsl_debug("SSL_accept failed %d / %s\n",
 						  m, ERR_error_string(m, NULL));
-
+go_again:
 		if (m == SSL_ERROR_WANT_READ) {
 			if (lws_change_pollfd(wsi, 0, LWS_POLLIN))
 				goto fail;