vh doubly linked list for wsi on same protocol

This trades off a couple of wsi pointers for vastly increased speed
for the callback when writeable "all protocol" variants when there
are many kinds of wsi active.

Signed-off-by: Andy Green <andy@warmcat.com>

lib/context.c

@@ -277,6 +277,9 @@ lws_create_vhost(struct lws_context *context,
 #endif
 		vh->protocols = info->protocols;
 
+	vh->same_vh_protocol_list = (struct lws **)
+			lws_zalloc(sizeof(struct lws *) * vh->count_protocols);
+
 	vh->mount_list = mounts;
 
 #ifdef LWS_USE_UNIX_SOCK
@@ -796,6 +799,7 @@ lws_context_destroy(struct lws_context *context)
 		if (vh->protocol_vh_privs)
 			lws_free(vh->protocol_vh_privs);
 		lws_ssl_SSL_CTX_destroy(vh);
+		lws_free(vh->same_vh_protocol_list);
 #ifdef LWS_WITH_PLUGINS
 		if (context->plugin_list)
 			lws_free((void *)vh->protocols);

lib/libuv.c

@@ -363,9 +363,9 @@ lws_libuv_stop(struct lws_context *context)
 
 		for (n = 0; (unsigned int)n < context->pt[m].fds_count; n++) {
 			struct lws *wsi = wsi_from_fd(context, pt->fds[n].fd);
+
 			if (!wsi)
 				continue;
-
 			lws_close_free_wsi(wsi,
 				LWS_CLOSE_STATUS_NOSTATUS_CONTEXT_DESTROY
 				/* no protocol close */);
@@ -373,6 +373,7 @@ lws_libuv_stop(struct lws_context *context)
 		}
 	}
 
+	lwsl_info("%s: feels everything closed\n", __func__);
 	if (context->count_wsi_allocated == 0)
 		lws_libuv_kill(context);
 }

lib/pollfd.c

@@ -191,6 +191,30 @@ remove_wsi_socket_from_fds(struct lws *wsi)
 					   wsi->user_space, (void *)&pa, 1))
 		return -1;
 
+	/*
+	 * detach ourselves from vh protocol list if we're on one
+	 * A -> B -> C
+	 * A -> C , or, B -> C, or A -> B
+	 */
+	lwsl_info("%s: removing same prot wsi %p\n", __func__, wsi);
+	if (wsi->same_vh_protocol_prev) {
+		assert (*(wsi->same_vh_protocol_prev) == wsi);
+		lwsl_info("have prev %p, setting him to our next %p\n",
+			 wsi->same_vh_protocol_prev,
+			 wsi->same_vh_protocol_next);
+
+		/* guy who pointed to us should point to our next */
+		*(wsi->same_vh_protocol_prev) = wsi->same_vh_protocol_next;
+	} //else
+		//lwsl_err("null wsi->prev\n");
+	/* our next should point back to our prev */
+	if (wsi->same_vh_protocol_next) {
+		lwsl_info("have next %p\n");
+		wsi->same_vh_protocol_next->same_vh_protocol_prev =
+				wsi->same_vh_protocol_prev;
+	} //else
+		//lwsl_err("null wsi->next\n");
+
 	lws_libev_io(wsi, LWS_EV_STOP | LWS_EV_READ | LWS_EV_WRITE | LWS_EV_PREPARE_DELETION);
 	lws_libuv_io(wsi, LWS_EV_STOP | LWS_EV_READ | LWS_EV_WRITE | LWS_EV_PREPARE_DELETION);
 
@@ -289,6 +313,9 @@ lws_callback_on_writable(struct lws *wsi)
 	if (wsi->state == LWSS_SHUTDOWN)
 		return 0;
 
+	if (wsi->socket_is_permanently_unusable)
+		return 0;
+
 #ifdef LWS_USE_HTTP2
 	lwsl_info("%s: %p\n", __func__, wsi);
 
@@ -348,45 +375,15 @@ network_sock:
 	return 1;
 }
 
-/**
- * lws_callback_on_writable_all_protocol() - Request a callback for
- *			all connections using the given protocol when it
- *			becomes possible to write to each socket without
- *			blocking in turn.
- *
- * @context:	lws_context
- * @protocol:	Protocol whose connections will get callbacks
- */
-
-LWS_VISIBLE int
-lws_callback_on_writable_all_protocol(const struct lws_context *context,
-				      const struct lws_protocols *protocol)
-{
-	const struct lws_context_per_thread *pt = &context->pt[0];
-	unsigned int n, m = context->count_threads;
-	struct lws *wsi;
-
-	while (m--) {
-		for (n = 0; n < pt->fds_count; n++) {
-			wsi = wsi_from_fd(context, pt->fds[n].fd);
-			if (!wsi)
-				continue;
-			if (wsi->protocol == protocol)
-				lws_callback_on_writable(wsi);
-		}
-		pt++;
-	}
-
-	return 0;
-}
-
-
 /**
  * lws_callback_on_writable_all_protocol_vhost() - Request a callback for
  *			all connections using the given protocol when it
  *			becomes possible to write to each socket without
  *			blocking in turn.
  *
+ *	This calls back connections with the same protocol ON THE SAME
+ *	VHOST ONLY.
+ *
  * @vhost:	Only consider connections on this lws_vhost
  * @protocol:	Protocol whose connections will get callbacks
  */
@@ -395,20 +392,66 @@ LWS_VISIBLE int
 lws_callback_on_writable_all_protocol_vhost(const struct lws_vhost *vhost,
 				      const struct lws_protocols *protocol)
 {
-	const struct lws_context *context = vhost->context;
-	const struct lws_context_per_thread *pt = &context->pt[0];
-	unsigned int n, m = context->count_threads;
 	struct lws *wsi;
 
-	while (m--) {
-		for (n = 0; n < pt->fds_count; n++) {
-			wsi = wsi_from_fd(context, pt->fds[n].fd);
-			if (!wsi)
-				continue;
-			if (wsi->vhost == vhost && wsi->protocol == protocol)
-				lws_callback_on_writable(wsi);
+	if (protocol < vhost->protocols ||
+	    protocol >= (vhost->protocols + vhost->count_protocols)) {
+		lwsl_err("%s: protocol is not from vhost\n", __func__);
+
+		return -1;
+	}
+
+	wsi = vhost->same_vh_protocol_list[protocol - vhost->protocols];
+	//lwsl_notice("%s: protocol %p, start wsi %p\n", __func__, protocol, wsi);
+	while (wsi) {
+		//lwsl_notice("%s: protocol %p, this wsi %p (wsi->protocol=%p)\n",
+		//		__func__, protocol, wsi, wsi->protocol);
+		assert(wsi->protocol == protocol);
+		assert(*wsi->same_vh_protocol_prev == wsi);
+		if (wsi->same_vh_protocol_next) {
+		//	lwsl_err("my next says %p\n", wsi->same_vh_protocol_next);
+		//	lwsl_err("my next's prev says %p\n",
+		//		wsi->same_vh_protocol_next->same_vh_protocol_prev);
+			assert(wsi->same_vh_protocol_next->same_vh_protocol_prev == &wsi->same_vh_protocol_next);
 		}
-		pt++;
+		lws_callback_on_writable(wsi);
+		wsi = wsi->same_vh_protocol_next;
+	}
+
+	return 0;
+}
+
+/**
+ * lws_callback_on_writable_all_protocol() - Request a callback for
+ *			all connections using the given protocol when it
+ *			becomes possible to write to each socket without
+ *			blocking in turn.
+ *
+ *	This calls back any connection using the same protocol on ANY
+ *	VHOST.
+ *
+ * @context:	lws_context
+ * @protocol:	Protocol whose connections will get callbacks
+ */
+
+LWS_VISIBLE int
+lws_callback_on_writable_all_protocol(const struct lws_context *context,
+				      const struct lws_protocols *protocol)
+{
+	struct lws_vhost *vhost = context->vhost_list;
+	int n;
+
+	while (vhost) {
+		for (n = 0; n < vhost->count_protocols; n++)
+			if (protocol->callback ==
+			    vhost->protocols[n].callback &&
+			    !strcmp(protocol->name, vhost->protocols[n].name))
+				break;
+		if (n != vhost->count_protocols)
+			lws_callback_on_writable_all_protocol_vhost(
+				vhost, &vhost->protocols[n]);
+
+		vhost = vhost->vhost_next;
 	}
 
 	return 0;

lib/private-libwebsockets.h

@@ -660,6 +660,7 @@ struct lws_vhost {
 	const struct lws_protocols *protocols;
 	void **protocol_vh_privs;
 	struct lws_protocol_vhost_options *pvo;
+	struct lws **same_vh_protocol_list;
 #ifdef LWS_OPENSSL_SUPPORT
 	SSL_CTX *ssl_ctx;
 	SSL_CTX *ssl_client_ctx;
@@ -1204,6 +1205,7 @@ struct lws {
 	struct lws_cgi *cgi; /* wsi being cgi master have one of these */
 #endif
 	const struct lws_protocols *protocol;
+	struct lws **same_vh_protocol_prev, *same_vh_protocol_next;
 	struct lws *timeout_list;
 	struct lws **timeout_list_prev;
 #ifdef LWS_WITH_ACCESS_LOG

lib/server.c

@@ -828,7 +828,7 @@ upgrade_ws:
 		hit = 0;
 
 		while (*p && !hit) {
-			unsigned int n = 0;
+			n = 0;
 			while (n < sizeof(protocol_name) - 1 && *p && *p !=',')
 				protocol_name[n++] = *p++;
 			protocol_name[n] = '\0';
@@ -842,7 +842,6 @@ upgrade_ws:
 				if (wsi->vhost->protocols[n].name &&
 				    !strcmp(wsi->vhost->protocols[n].name,
 					    protocol_name)) {
-					lwsl_info("prot match %d\n", n);
 					wsi->protocol = &wsi->vhost->protocols[n];
 					hit = 1;
 					break;
@@ -862,10 +861,11 @@ upgrade_ws:
 			}
 			/*
 			 * some clients only have one protocol and
-			 * do not sent the protocol list header...
+			 * do not send the protocol list header...
 			 * allow it and match to protocol 0
 			 */
 			lwsl_info("defaulting to prot 0 handler\n");
+			n = 0;
 			wsi->protocol = &wsi->vhost->protocols[0];
 		}
 
@@ -877,7 +877,6 @@ upgrade_ws:
 		 * Give the user code a chance to study the request and
 		 * have the opportunity to deny it
 		 */
-
 		if ((wsi->protocol->callback)(wsi,
 				LWS_CALLBACK_FILTER_PROTOCOL_CONNECTION,
 				wsi->user_space,
@@ -906,6 +905,31 @@ upgrade_ws:
 			goto bail_nuke_ah;
 		}
 
+		/*
+		 * stitch protocol choice into the vh protocol linked list
+		 * We always insert ourselves at the start of the list
+		 *
+		 * X <-> B
+		 * X <-> pAn <-> pB
+		 */
+		//lwsl_err("%s: pre insert vhost start wsi %p, that wsi prev == %p\n",
+		//		__func__,
+		//		wsi->vhost->same_vh_protocol_list[n],
+		//		wsi->same_vh_protocol_prev);
+		wsi->same_vh_protocol_prev = /* guy who points to us */
+			&wsi->vhost->same_vh_protocol_list[n];
+		wsi->same_vh_protocol_next = /* old first guy is our next */
+				wsi->vhost->same_vh_protocol_list[n];
+		/* we become the new first guy */
+		wsi->vhost->same_vh_protocol_list[n] = wsi;
+
+		if (wsi->same_vh_protocol_next)
+			/* old first guy points back to us now */
+			wsi->same_vh_protocol_next->same_vh_protocol_prev =
+					&wsi->same_vh_protocol_next;
+
+
+
 		/* we are upgrading to ws, so http/1.1 and keepalive +
 		 * pipelined header considerations about keeping the ah around
 		 * no longer apply.  However it's common for the first ws

plugins/protocol_dumb_increment.c

@@ -40,7 +40,7 @@ uv_timeout_cb_dumb_increment(uv_timer_t *w
 {
 	struct per_vhost_data__dumb_increment *vhd = lws_container_of(w,
 			struct per_vhost_data__dumb_increment, timeout_watcher);
-	lws_callback_on_writable_all_protocol(vhd->context, vhd->protocol);
+	lws_callback_on_writable_all_protocol_vhost(vhd->vhost, vhd->protocol);
 }
 
 static int