vh doubly linked list for wsi on same protocol

This trades off a couple of wsi pointers for vastly increased speed
for the callback when writeable "all protocol" variants when there
are many kinds of wsi active.

Signed-off-by: Andy Green <andy@warmcat.com>

lib/context.c

@@ -277,6 +277,9 @@ lws_create_vhost(struct lws_context *context,
 #endif
 		vh->protocols = info->protocols;
 
+	vh->same_vh_protocol_list = (struct lws **)
+			lws_zalloc(sizeof(struct lws *) * vh->count_protocols);
+
 	vh->mount_list = mounts;
 
 #ifdef LWS_USE_UNIX_SOCK
@@ -796,6 +799,7 @@ lws_context_destroy(struct lws_context *context)
 		if (vh->protocol_vh_privs)
 			lws_free(vh->protocol_vh_privs);
 		lws_ssl_SSL_CTX_destroy(vh);
+		lws_free(vh->same_vh_protocol_list);
 #ifdef LWS_WITH_PLUGINS
 		if (context->plugin_list)
 			lws_free((void *)vh->protocols);

lib/libuv.c

@@ -363,9 +363,9 @@ lws_libuv_stop(struct lws_context *context)
 
 		for (n = 0; (unsigned int)n < context->pt[m].fds_count; n++) {
 			struct lws *wsi = wsi_from_fd(context, pt->fds[n].fd);
+
 			if (!wsi)
 				continue;
-
 			lws_close_free_wsi(wsi,
 				LWS_CLOSE_STATUS_NOSTATUS_CONTEXT_DESTROY
 				/* no protocol close */);
@@ -373,6 +373,7 @@ lws_libuv_stop(struct lws_context *context)
 		}
 	}
 
+	lwsl_err("%s: feels everything closed\n", __func__);
 	if (context->count_wsi_allocated == 0)
 		lws_libuv_kill(context);
 }

lib/libwebsockets.c

@@ -400,6 +400,17 @@ just_kill_connection:
 		wsi->rw = NULL;
 	}
 #endif
+	/*
+	 * detach ourselves from vh protocol list if we're on one
+	 */
+	if (wsi->same_vh_protocol_prev) {
+		/* guy who pointed to us should point to our next */
+		*(wsi->same_vh_protocol_prev) = wsi->same_vh_protocol_next;
+		/* our next should point back to our prev */
+		if (wsi->same_vh_protocol_next)
+			wsi->same_vh_protocol_next->same_vh_protocol_prev =
+				wsi->same_vh_protocol_prev;
+	}
 	/*
 	 * we won't be servicing or receiving anything further from this guy
 	 * delete socket from the internal poll list if still present

lib/pollfd.c

@@ -289,6 +289,9 @@ lws_callback_on_writable(struct lws *wsi)
 	if (wsi->state == LWSS_SHUTDOWN)
 		return 0;
 
+	if (wsi->socket_is_permanently_unusable)
+		return 0;
+
 #ifdef LWS_USE_HTTP2
 	lwsl_info("%s: %p\n", __func__, wsi);
 
@@ -348,45 +351,15 @@ network_sock:
 	return 1;
 }
 
-/**
- * lws_callback_on_writable_all_protocol() - Request a callback for
- *			all connections using the given protocol when it
- *			becomes possible to write to each socket without
- *			blocking in turn.
- *
- * @context:	lws_context
- * @protocol:	Protocol whose connections will get callbacks
- */
-
-LWS_VISIBLE int
-lws_callback_on_writable_all_protocol(const struct lws_context *context,
-				      const struct lws_protocols *protocol)
-{
-	const struct lws_context_per_thread *pt = &context->pt[0];
-	unsigned int n, m = context->count_threads;
-	struct lws *wsi;
-
-	while (m--) {
-		for (n = 0; n < pt->fds_count; n++) {
-			wsi = wsi_from_fd(context, pt->fds[n].fd);
-			if (!wsi)
-				continue;
-			if (wsi->protocol == protocol)
-				lws_callback_on_writable(wsi);
-		}
-		pt++;
-	}
-
-	return 0;
-}
-
-
 /**
  * lws_callback_on_writable_all_protocol_vhost() - Request a callback for
  *			all connections using the given protocol when it
  *			becomes possible to write to each socket without
  *			blocking in turn.
  *
+ *	This calls back connections with the same protocol ON THE SAME
+ *	VHOST ONLY.
+ *
  * @vhost:	Only consider connections on this lws_vhost
  * @protocol:	Protocol whose connections will get callbacks
  */
@@ -395,20 +368,56 @@ LWS_VISIBLE int
 lws_callback_on_writable_all_protocol_vhost(const struct lws_vhost *vhost,
 				      const struct lws_protocols *protocol)
 {
-	const struct lws_context *context = vhost->context;
-	const struct lws_context_per_thread *pt = &context->pt[0];
-	unsigned int n, m = context->count_threads;
 	struct lws *wsi;
 
-	while (m--) {
-		for (n = 0; n < pt->fds_count; n++) {
-			wsi = wsi_from_fd(context, pt->fds[n].fd);
-			if (!wsi)
-				continue;
-			if (wsi->vhost == vhost && wsi->protocol == protocol)
-				lws_callback_on_writable(wsi);
-		}
-		pt++;
+	if (protocol < vhost->protocols ||
+	    protocol >= (vhost->protocols + vhost->count_protocols)) {
+		lwsl_err("%s: protocol is not from vhost\n", __func__);
+
+		return -1;
+	}
+
+	wsi = vhost->same_vh_protocol_list[protocol - vhost->protocols];
+	while (wsi) {
+		assert(wsi->protocol == protocol);
+		lws_callback_on_writable(wsi);
+		wsi = wsi->same_vh_protocol_next;
+	}
+
+	return 0;
+}
+
+/**
+ * lws_callback_on_writable_all_protocol() - Request a callback for
+ *			all connections using the given protocol when it
+ *			becomes possible to write to each socket without
+ *			blocking in turn.
+ *
+ *	This calls back any connection using the same protocol on ANY
+ *	VHOST.
+ *
+ * @context:	lws_context
+ * @protocol:	Protocol whose connections will get callbacks
+ */
+
+LWS_VISIBLE int
+lws_callback_on_writable_all_protocol(const struct lws_context *context,
+				      const struct lws_protocols *protocol)
+{
+	struct lws_vhost *vhost = context->vhost_list;
+	int n;
+
+	while (vhost) {
+		for (n = 0; n < vhost->count_protocols; n++)
+			if (protocol->callback ==
+			    vhost->protocols[n].callback &&
+			    !strcmp(protocol->name, vhost->protocols[n].name))
+				break;
+		if (n != vhost->count_protocols)
+			lws_callback_on_writable_all_protocol_vhost(
+				vhost, &vhost->protocols[n]);
+
+		vhost = vhost->vhost_next;
 	}
 
 	return 0;

lib/private-libwebsockets.h

@@ -660,6 +660,7 @@ struct lws_vhost {
 	const struct lws_protocols *protocols;
 	void **protocol_vh_privs;
 	struct lws_protocol_vhost_options *pvo;
+	struct lws **same_vh_protocol_list;
 #ifdef LWS_OPENSSL_SUPPORT
 	SSL_CTX *ssl_ctx;
 	SSL_CTX *ssl_client_ctx;
@@ -1204,6 +1205,7 @@ struct lws {
 	struct lws_cgi *cgi; /* wsi being cgi master have one of these */
 #endif
 	const struct lws_protocols *protocol;
+	struct lws **same_vh_protocol_prev, *same_vh_protocol_next;
 	struct lws *timeout_list;
 	struct lws **timeout_list_prev;
 #ifdef LWS_WITH_ACCESS_LOG

lib/server.c

@@ -828,7 +828,7 @@ upgrade_ws:
 		hit = 0;
 
 		while (*p && !hit) {
-			unsigned int n = 0;
+			n = 0;
 			while (n < sizeof(protocol_name) - 1 && *p && *p !=',')
 				protocol_name[n++] = *p++;
 			protocol_name[n] = '\0';
@@ -842,7 +842,6 @@ upgrade_ws:
 				if (wsi->vhost->protocols[n].name &&
 				    !strcmp(wsi->vhost->protocols[n].name,
 					    protocol_name)) {
-					lwsl_info("prot match %d\n", n);
 					wsi->protocol = &wsi->vhost->protocols[n];
 					hit = 1;
 					break;
@@ -866,6 +865,7 @@ upgrade_ws:
 			 * allow it and match to protocol 0
 			 */
 			lwsl_info("defaulting to prot 0 handler\n");
+			n = 0;
 			wsi->protocol = &wsi->vhost->protocols[0];
 		}
 
@@ -877,7 +877,6 @@ upgrade_ws:
 		 * Give the user code a chance to study the request and
 		 * have the opportunity to deny it
 		 */
-
 		if ((wsi->protocol->callback)(wsi,
 				LWS_CALLBACK_FILTER_PROTOCOL_CONNECTION,
 				wsi->user_space,
@@ -886,6 +885,16 @@ upgrade_ws:
 			goto bail_nuke_ah;
 		}
 
+		/*
+		 * stitch protocol choice into the vh protocol linked list
+		 */
+		wsi->same_vh_protocol_prev = /* guy who points to us */
+			&wsi->vhost->same_vh_protocol_list[n];
+		wsi->same_vh_protocol_next = /* old first guy is our next */
+				wsi->vhost->same_vh_protocol_list[n];
+		/* we become the new first guy */
+		wsi->vhost->same_vh_protocol_list[n] = wsi;
+
 		/*
 		 * Perform the handshake according to the protocol version the
 		 * client announced

plugins/protocol_dumb_increment.c

@@ -40,7 +40,7 @@ uv_timeout_cb_dumb_increment(uv_timer_t *w
 {
 	struct per_vhost_data__dumb_increment *vhd = lws_container_of(w,
 			struct per_vhost_data__dumb_increment, timeout_watcher);
-	lws_callback_on_writable_all_protocol(vhd->context, vhd->protocol);
+	lws_callback_on_writable_all_protocol_vhost(vhd->vhost, vhd->protocol);
 }
 
 static int