diff --git a/lib/core-net/wsi.c b/lib/core-net/wsi.c
index ddf389609..953b7477b 100644
--- a/lib/core-net/wsi.c
+++ b/lib/core-net/wsi.c
@@ -635,7 +635,10 @@ lws_wsi_user(struct lws *wsi)
 void
 lws_set_wsi_user(struct lws *wsi, void *data)
 {
-	if (wsi->user_space_externally_allocated)
+	    /* client and user owns it... */
+	if (wsi->user_space_externally_allocated ||
+	    /* server, and protocol can never own it, user can control it... */
+	    (wsi->protocol && !wsi->protocol->per_session_data_size))
 		wsi->user_space = data;
 	else
 		lwsl_err("%s: Cannot set internally-allocated user_space\n",