From cfeb196479f29294c05cf0e4dd0ecef7d0043405 Mon Sep 17 00:00:00 2001 From: Andy Green Date: Wed, 25 Jul 2018 16:26:41 +0800 Subject: [PATCH] client: SMP: associate client with tsi of thread creating client connection 1) This moves the service tid detection stuff from context to pt. 2) If LWS_MAX_SMP > 1, a default pthread tid detection callback is provided on the dummy callback. Callback handlers that call through to the dummy handler will inherit this. It provides an int truncation of the pthread tid. 3) If there has been any service calls on the service threads, the pts now know the low sizeof(int) bytes of their tid. When you ask for a client connection to be created, it looks through the pts to see if the calling thread is a pt service thread. If so, the new client is set to use the same pt as the caller. --- lib/core/connect.c | 78 +++++++++++++++++++++--------- lib/core/context.c | 5 ++ lib/core/pollfd.c | 3 +- lib/core/private.h | 17 ++++--- lib/libwebsockets.h | 12 ++--- lib/plat/esp32/esp32-service.c | 8 ++- lib/plat/optee/lws-plat-optee.c | 8 ++- lib/plat/unix/unix-service.c | 7 ++- lib/plat/windows/windows-service.c | 9 ++-- 9 files changed, 88 insertions(+), 59 deletions(-) diff --git a/lib/core/connect.c b/lib/core/connect.c index f1c6bf4b8..1bcb50f01 100644 --- a/lib/core/connect.c +++ b/lib/core/connect.c @@ -45,6 +45,9 @@ lws_client_connect_via_info(const struct lws_client_connect_info *i) struct lws *wsi; const struct lws_protocols *p; const char *local = i->protocol; +#if LWS_MAX_SMP > 1 + int n, tid; +#endif if (i->context->requested_kill) return NULL; @@ -67,27 +70,6 @@ lws_client_connect_via_info(const struct lws_client_connect_info *i) wsi->context = i->context; wsi->desc.sockfd = LWS_SOCK_INVALID; - /* - * PHASE 2: Choose an initial role for the wsi and do role-specific init - * - * Note the initial role may not reflect the final role, eg, - * we may want ws, but first we have to go through h1 to get that - */ - - lws_role_call_client_bind(wsi, i); - - /* - * PHASE 3: fill up the wsi with stuff from the connect_info as far as - * it can go. It's uncertain because not only is our connection - * going to complete asynchronously, we might have bound to h1 and not - * even be able to get ahold of an ah immediately. - */ - - wsi->user_space = NULL; - wsi->pending_timeout = NO_PENDING_TIMEOUT; - wsi->position_in_fds_table = LWS_NO_FDS_POS; - wsi->c_port = i->port; - wsi->vhost = NULL; if (!i->vhost) lws_vhost_bind_wsi(i->context->vhost_list, wsi); @@ -100,6 +82,54 @@ lws_client_connect_via_info(const struct lws_client_connect_info *i) goto bail; } + /* + * PHASE 2: if SMP, bind the client to whatever tsi the current thread + * represents + */ + +#if LWS_MAX_SMP > 1 + tid = wsi->vhost->protocols[0].callback(wsi, + LWS_CALLBACK_GET_THREAD_ID, NULL, NULL, 0); + + lws_context_lock(i->context, "client find tsi"); + + for (n = 0; n < i->context->count_threads; n++) + if (i->context->pt[n].service_tid == tid) { + lwsl_info("%s: client binds to caller tsi %d\n", + __func__, n); + wsi->tsi = n; + break; + } + + /* + * this binding is sort of provisional, since when we try to insert + * into the pt fds, there may be no space and it will fail + */ + + lws_context_unlock(i->context); +#endif + + /* + * PHASE 3: Choose an initial role for the wsi and do role-specific init + * + * Note the initial role may not reflect the final role, eg, + * we may want ws, but first we have to go through h1 to get that + */ + + lws_role_call_client_bind(wsi, i); + + /* + * PHASE 4: fill up the wsi with stuff from the connect_info as far as + * it can go. It's uncertain because not only is our connection + * going to complete asynchronously, we might have bound to h1 and not + * even be able to get ahold of an ah immediately. + */ + + wsi->user_space = NULL; + wsi->pending_timeout = NO_PENDING_TIMEOUT; + wsi->position_in_fds_table = LWS_NO_FDS_POS; + wsi->c_port = i->port; + wsi->protocol = &wsi->vhost->protocols[0]; wsi->client_pipeline = !!(i->ssl_connection & LCCSCF_PIPELINE); @@ -111,7 +141,7 @@ lws_client_connect_via_info(const struct lws_client_connect_info *i) } /* - * PHASE 4: handle external user_space now, generic alloc is done in + * PHASE 5: handle external user_space now, generic alloc is done in * role finalization */ @@ -130,7 +160,7 @@ lws_client_connect_via_info(const struct lws_client_connect_info *i) #endif /* - * PHASE 5: stash the things from connect_info that we can't process + * PHASE 6: stash the things from connect_info that we can't process * right now, eg, if http binding, without an ah. If h1 and no ah, we * will go on the ah waiting list and process those things later (after * the connect_info and maybe the things pointed to have gone out of @@ -180,7 +210,7 @@ lws_client_connect_via_info(const struct lws_client_connect_info *i) } /* - * PHASE 6: Do any role-specific finalization processing. We can still + * PHASE 7: Do any role-specific finalization processing. We can still * see important info things via wsi->stash */ diff --git a/lib/core/context.c b/lib/core/context.c index ebc1fb84c..5557b814f 100644 --- a/lib/core/context.c +++ b/lib/core/context.c @@ -667,6 +667,11 @@ lws_callback_http_dummy(struct lws *wsi, enum lws_callback_reasons reason, si->where, si->ret); break; +#if LWS_MAX_SMP > 1 + case LWS_CALLBACK_GET_THREAD_ID: + return (int)(unsigned long long)pthread_self(); +#endif + default: break; } diff --git a/lib/core/pollfd.c b/lib/core/pollfd.c index 24f7c7ab7..514ea255e 100644 --- a/lib/core/pollfd.c +++ b/lib/core/pollfd.c @@ -182,7 +182,7 @@ _lws_change_pollfd(struct lws *wsi, int _and, int _or, struct lws_pollargs *pa) ret = -1; goto bail; } - sampled_tid = context->service_tid; + sampled_tid = pt->service_tid; if (sampled_tid && wsi->vhost) { tid = wsi->vhost->protocols[0].callback(wsi, LWS_CALLBACK_GET_THREAD_ID, NULL, NULL, 0); @@ -335,6 +335,7 @@ __remove_wsi_socket_from_fds(struct lws *wsi) pt->fds[m] = pt->fds[pt->fds_count - 1]; /* this decrements pt->fds_count */ lws_plat_delete_socket_from_fds(context, wsi, m); + pt->count_conns--; v = (int) pt->fds[m].fd; /* end guy's "position in fds table" is now the deletion guy's old one */ end_wsi = wsi_from_fd(context, v); diff --git a/lib/core/private.h b/lib/core/private.h index 28b310c15..de78bf27a 100644 --- a/lib/core/private.h +++ b/lib/core/private.h @@ -394,6 +394,15 @@ struct lws_context_per_thread { unsigned long count_conns; unsigned int fds_count; + /* + * set to the Thread ID that's doing the service loop just before entry + * to poll indicates service thread likely idling in poll() + * volatile because other threads may check it as part of processing + * for pollfd event change. + */ + volatile int service_tid; + int service_tid_detected; + volatile unsigned char inside_poll; volatile unsigned char foreign_spinlock; @@ -691,14 +700,6 @@ struct lws_context { unsigned int doing_protocol_init:1; unsigned int done_protocol_destroy_cb:1; unsigned int finalize_destroy_after_internal_loops_stopped:1; - /* - * set to the Thread ID that's doing the service loop just before entry - * to poll indicates service thread likely idling in poll() - * volatile because other threads may check it as part of processing - * for pollfd event change. - */ - volatile int service_tid; - int service_tid_detected; short count_threads; short plugin_protocol_count; diff --git a/lib/libwebsockets.h b/lib/libwebsockets.h index ad074ec30..273f835e6 100644 --- a/lib/libwebsockets.h +++ b/lib/libwebsockets.h @@ -3491,6 +3491,12 @@ struct lws_client_connect_info { * non-NULL: this protocol name is used to bind the connection to * the local protocol handler. .protocol is used for the * list of remote ws protocols we could accept */ + const char *alpn; + /**< NULL: allow lws default ALPN list, from vhost if present or from + * list of roles built into lws + * non-NULL: require one from provided comma-separated list of alpn + * tokens + */ /* Add new things just above here ---^ * This is part of the ABI, don't needlessly break compatibility @@ -3499,12 +3505,6 @@ struct lws_client_connect_info { * members added above will see 0 (default) even if the app * was not built against the newer headers. */ - const char *alpn; - /* NULL: allow lws default ALPN list, from vhost if present or from - * list of roles built into lws - * non-NULL: require one from provided comma-separated list of alpn - * tokens - */ void *_unused[4]; /**< dummy */ }; diff --git a/lib/plat/esp32/esp32-service.c b/lib/plat/esp32/esp32-service.c index 3549d9d49..fcf3488eb 100644 --- a/lib/plat/esp32/esp32-service.c +++ b/lib/plat/esp32/esp32-service.c @@ -76,16 +76,14 @@ _lws_plat_service_tsi(struct lws_context *context, int timeout_ms, int tsi) if (timeout_ms < 0) goto faked_service; - if (!context->service_tid_detected) { + if (!pt->service_tid_detected) { struct lws *_lws = lws_zalloc(sizeof(*_lws), "tid probe"); _lws->context = context; - context->service_tid_detected = - context->vhost_list->protocols[0].callback( + pt->service_tid = context->vhost_list->protocols[0].callback( _lws, LWS_CALLBACK_GET_THREAD_ID, NULL, NULL, 0); - context->service_tid = context->service_tid_detected; - context->service_tid_detected = 1; + pt->service_tid_detected = 1; lws_free(_lws); } diff --git a/lib/plat/optee/lws-plat-optee.c b/lib/plat/optee/lws-plat-optee.c index d5ba9017f..9803dad49 100644 --- a/lib/plat/optee/lws-plat-optee.c +++ b/lib/plat/optee/lws-plat-optee.c @@ -110,17 +110,15 @@ _lws_plat_service_tsi(struct lws_context *context, int timeout_ms, int tsi) if (timeout_ms < 0) goto faked_service; - if (!context->service_tid_detected) { + if (!pt->service_tid_detected) { struct lws _lws; memset(&_lws, 0, sizeof(_lws)); _lws.context = context; - context->service_tid_detected = - context->vhost_list->protocols[0].callback( + pt->service_tid = context->vhost_list->protocols[0].callback( &_lws, LWS_CALLBACK_GET_THREAD_ID, NULL, NULL, 0); - context->service_tid = context->service_tid_detected; - context->service_tid_detected = 1; + pt->service_tid_detected = 1; } /* diff --git a/lib/plat/unix/unix-service.c b/lib/plat/unix/unix-service.c index 9a7b84f09..00c1fc926 100644 --- a/lib/plat/unix/unix-service.c +++ b/lib/plat/unix/unix-service.c @@ -52,17 +52,16 @@ _lws_plat_service_tsi(struct lws_context *context, int timeout_ms, int tsi) if (context->event_loop_ops->run_pt) context->event_loop_ops->run_pt(context, tsi); - if (!context->service_tid_detected) { + if (!pt->service_tid_detected) { struct lws _lws; memset(&_lws, 0, sizeof(_lws)); _lws.context = context; - context->service_tid_detected = + pt->service_tid = context->vhost_list->protocols[0].callback( &_lws, LWS_CALLBACK_GET_THREAD_ID, NULL, NULL, 0); - context->service_tid = context->service_tid_detected; - context->service_tid_detected = 1; + pt->service_tid_detected = 1; } /* diff --git a/lib/plat/windows/windows-service.c b/lib/plat/windows/windows-service.c index 9351da031..b027adddd 100644 --- a/lib/plat/windows/windows-service.c +++ b/lib/plat/windows/windows-service.c @@ -42,17 +42,16 @@ _lws_plat_service_tsi(struct lws_context *context, int timeout_ms, int tsi) pt = &context->pt[tsi]; - if (!context->service_tid_detected) { + if (!pt->service_tid_detected) { struct lws _lws; memset(&_lws, 0, sizeof(_lws)); _lws.context = context; - context->service_tid_detected = context->vhost_list-> + pt->service_tid = context->vhost_list-> protocols[0].callback(&_lws, LWS_CALLBACK_GET_THREAD_ID, NULL, NULL, 0); - context->service_tid = context->service_tid_detected; - context->service_tid_detected = 1; + pt->service_tid_detected = 1; } if (timeout_ms < 0) { @@ -173,8 +172,6 @@ _lws_plat_service_tsi(struct lws_context *context, int timeout_ms, int tsi) } } - context->service_tid = 0; - if (ev == WSA_WAIT_TIMEOUT) lws_service_fd(context, NULL);