diff --git a/lib/core/adopt.c b/lib/core/adopt.c index 4cf4d89cc..5eafefd2d 100644 --- a/lib/core/adopt.c +++ b/lib/core/adopt.c @@ -64,7 +64,7 @@ lws_create_new_server_wsi(struct lws_vhost *vhost, int fixed_tsi) lwsl_debug("new wsi %p joining vhost %s, tsi %d\n", new_wsi, vhost->name, new_wsi->tsi); - new_wsi->vhost = vhost; + lws_vhost_bind_wsi(vhost, new_wsi); new_wsi->context = vhost->context; new_wsi->pending_timeout = NO_PENDING_TIMEOUT; new_wsi->rxflow_change_to = LWS_RXFLOW_ALLOW; @@ -241,8 +241,10 @@ bail: parent->child_list = new_wsi->sibling_list; if (new_wsi->user_space) lws_free(new_wsi->user_space); + lws_vhost_unbind_wsi(new_wsi); lws_free(new_wsi); - compatible_close(fd.sockfd); + + compatible_close(fd.sockfd); return NULL; } diff --git a/lib/core/connect.c b/lib/core/connect.c index ace3dc18e..4de7dff4e 100644 --- a/lib/core/connect.c +++ b/lib/core/connect.c @@ -97,6 +97,8 @@ lws_client_connect_via_info(const struct lws_client_connect_info *i) goto bail; } + lws_vhost_bind_wsi(wsi->vhost, wsi); + wsi->protocol = &wsi->vhost->protocols[0]; wsi->client_pipeline = !!(i->ssl_connection & LCCSCF_PIPELINE); diff --git a/lib/core/context.c b/lib/core/context.c index b7caa7966..029313b35 100644 --- a/lib/core/context.c +++ b/lib/core/context.c @@ -698,9 +698,6 @@ static const struct lws_protocols protocols_dummy[] = { #undef LWS_HAVE_GETENV #endif -static void -lws_vhost_destroy2(struct lws_vhost *vh); - LWS_VISIBLE struct lws_vhost * lws_create_vhost(struct lws_context *context, const struct lws_context_creation_info *info) @@ -759,6 +756,8 @@ lws_create_vhost(struct lws_context *context, vh->pvo = info->pvo; vh->headers = info->headers; vh->user = info->user; + vh->finalize = info->finalize; + vh->finalize_arg = info->finalize_arg; LWS_FOR_EVERY_AVAILABLE_ROLE_START(ar) if (ar->init_vhost) @@ -1025,8 +1024,7 @@ lws_create_vhost(struct lws_context *context, return vh; bail1: - lws_vhost_destroy(vh); - lws_vhost_destroy2(vh); + lws_vhost_destroy(vh, NULL, NULL); return NULL; @@ -1614,24 +1612,27 @@ lws_context_is_deprecated(struct lws_context *context) void lws_vhost_destroy1(struct lws_vhost *vh) { - const struct lws_protocols *protocol = NULL; - struct lws_context_per_thread *pt; - int n, m = vh->context->count_threads; struct lws_context *context = vh->context; - struct lws wsi; lwsl_info("%s\n", __func__); if (vh->being_destroyed) return; + lws_vhost_lock(vh); /* -------------- vh { */ + vh->being_destroyed = 1; /* + * PHASE 1: take down or reassign any listen wsi + * * Are there other vhosts that are piggybacking on our listen socket? * If so we need to hand the listen socket off to one of the others - * so it will remain open. If not, leave it attached to the closing - * vhost and it will get closed. + * so it will remain open. + * + * If not, leave it attached to the closing vhost, the vh being marked + * being_destroyed will defeat any service and it will get closed in + * later phases. */ if (vh->lserv_wsi) @@ -1652,9 +1653,11 @@ lws_vhost_destroy1(struct lws_vhost *vh) */ assert(v->lserv_wsi == NULL); v->lserv_wsi = vh->lserv_wsi; - vh->lserv_wsi = NULL; - if (v->lserv_wsi) - v->lserv_wsi->vhost = v; + + if (v->lserv_wsi) { + lws_vhost_unbind_wsi(vh->lserv_wsi); + lws_vhost_bind_wsi(v, v->lserv_wsi); + } lwsl_notice("%s: listen skt from %s to %s\n", __func__, vh->name, v->name); @@ -1662,29 +1665,25 @@ lws_vhost_destroy1(struct lws_vhost *vh) } } lws_end_foreach_ll(v, vhost_next); + lws_vhost_unlock(vh); /* } vh -------------- */ + /* - * Forcibly close every wsi assoicated with this vhost. That will - * include the listen socket if it is still associated with the closing - * vhost. + * lws_check_deferred_free() will notice there is a vhost that is + * marked for destruction during the next 1s, for all tsi. + * + * It will start closing all wsi on this vhost. When the last wsi + * is closed, it will trigger lws_vhost_destroy2() */ +} - while (m--) { - pt = &context->pt[m]; - - for (n = 0; (unsigned int)n < context->pt[m].fds_count; n++) { - struct lws *wsi = wsi_from_fd(context, pt->fds[n].fd); - if (!wsi) - continue; - if (wsi->vhost != vh) - continue; - - lws_close_free_wsi(wsi, - LWS_CLOSE_STATUS_NOSTATUS_CONTEXT_DESTROY, - "vh destroy" - /* no protocol close */); - n--; - } - } +void +lws_vhost_destroy2(struct lws_vhost *vh) +{ + const struct lws_protocols *protocol = NULL; + struct lws_context *context = vh->context; + struct lws_deferred_free *df; + struct lws wsi; + int n; /* * destroy any pending timed events @@ -1699,7 +1698,7 @@ lws_vhost_destroy1(struct lws_vhost *vh) memset(&wsi, 0, sizeof(wsi)); wsi.context = vh->context; - wsi.vhost = vh; + wsi.vhost = vh; /* not a real bound wsi */ protocol = vh->protocols; if (protocol && vh->created_vhost_protocols) { n = 0; @@ -1727,15 +1726,6 @@ lws_vhost_destroy1(struct lws_vhost *vh) vh->vhost_next = vh->context->vhost_pending_destruction_list; vh->context->vhost_pending_destruction_list = vh; -} - -static void -lws_vhost_destroy2(struct lws_vhost *vh) -{ - const struct lws_protocols *protocol = NULL; - struct lws_context *context = vh->context; - struct lws_deferred_free *df; - int n; lwsl_info("%s: %p\n", __func__, vh); @@ -1819,31 +1809,74 @@ lws_vhost_destroy2(struct lws_vhost *vh) * they do not refer to the vhost. So it's safe to free. */ + if (vh->finalize) + vh->finalize(vh, vh->finalize_arg); + lwsl_info(" %s: Freeing vhost %p\n", __func__, vh); memset(vh, 0, sizeof(*vh)); lws_free(vh); } -int -lws_check_deferred_free(struct lws_context *context, int force) -{ - struct lws_deferred_free *df; - time_t now = lws_now_secs(); +/* + * each service thread calls this once a second or so + */ - lws_start_foreach_llp(struct lws_deferred_free **, pdf, - context->deferred_free_list) { - if (force || - lws_compare_time_t(context, now, (*pdf)->deadline) > 5) { - df = *pdf; - *pdf = df->next; - /* finalize vh destruction */ - lwsl_notice("deferred vh %p destroy\n", df->payload); - lws_vhost_destroy2(df->payload); - lws_free(df); - continue; /* after deletion we already point to next */ +int +lws_check_deferred_free(struct lws_context *context, int tsi, int force) +{ + struct lws_context_per_thread *pt; + int n; + + /* + * If we see a vhost is being destroyed, forcibly close every wsi on + * this tsi associated with this vhost. That will include the listen + * socket if it is still associated with the closing vhost. + * + * For SMP, we do this once per tsi per destroyed vhost. The reference + * counting on the vhost as the bound wsi close will notice that there + * are no bound wsi left, that vhost destruction can complete, + * and perform it. It doesn't matter which service thread does that + * because there is nothing left using the vhost to conflict. + */ + + lws_context_lock(context); /* ------------------- context { */ + + lws_start_foreach_ll(struct lws_vhost *, v, context->vhost_list) { + if (v->being_destroyed +#if LWS_MAX_SMP > 1 + && !v->close_flow_vs_tsi[tsi] +#endif + ) { + + pt = &context->pt[tsi]; + + lws_pt_lock(pt, "vhost removal"); /* -------------- pt { */ + +#if LWS_MAX_SMP > 1 + v->close_flow_vs_tsi[tsi] = 1; +#endif + + for (n = 0; (unsigned int)n < pt->fds_count; n++) { + struct lws *wsi = wsi_from_fd(context, pt->fds[n].fd); + if (!wsi) + continue; + if (wsi->vhost != v) + continue; + + __lws_close_free_wsi(wsi, + LWS_CLOSE_STATUS_NOSTATUS_CONTEXT_DESTROY, + "vh destroy" + /* no protocol close */); + n--; + } + + lws_pt_unlock(pt); /* } pt -------------- */ } - } lws_end_foreach_llp(pdf, next); + } lws_end_foreach_ll(v, vhost_next); + + + lws_context_unlock(context); /* } context ------------------- */ return 0; } @@ -1858,6 +1891,20 @@ lws_vhost_destroy(struct lws_vhost *vh) lws_vhost_destroy1(vh); + if (!vh->count_bound_wsi) { + /* + * After listen handoff, there are already no wsi bound to this + * vhost by any pt: nothing can be servicing any wsi belonging + * to it any more. + * + * Finalize the vh destruction immediately + */ + lws_vhost_destroy2(vh); + lws_free(df); + + return; + } + /* part 2 is deferred to allow all the handle closes to complete */ df->next = vh->context->deferred_free_list; @@ -1980,7 +2027,7 @@ lws_context_destroy2(struct lws_context *context) if (context->external_baggage_free_on_destroy) free(context->external_baggage_free_on_destroy); - lws_check_deferred_free(context, 1); + lws_check_deferred_free(context, 0, 1); #if LWS_MAX_SMP > 1 pthread_mutex_destroy(&context->lock); diff --git a/lib/core/libwebsockets.c b/lib/core/libwebsockets.c index 868c65aba..e5325895f 100644 --- a/lib/core/libwebsockets.c +++ b/lib/core/libwebsockets.c @@ -88,6 +88,40 @@ signed char char_to_hex(const char c) return -1; } +void +lws_vhost_bind_wsi(struct lws_vhost *vh, struct lws *wsi) +{ + wsi->vhost = vh; + vh->count_bound_wsi++; + lwsl_info("%s: vh %s: count_bound_wsi %d\n", + __func__, vh->name, vh->count_bound_wsi); + assert(wsi->vhost->count_bound_wsi > 0); +} + +void +lws_vhost_unbind_wsi(struct lws *wsi) +{ + if (wsi->vhost) { + assert(wsi->vhost->count_bound_wsi > 0); + wsi->vhost->count_bound_wsi--; + lwsl_info("%s: vh %s: count_bound_wsi %d\n", __func__, + wsi->vhost->name, wsi->vhost->count_bound_wsi); + + if (!wsi->vhost->count_bound_wsi && + wsi->vhost->being_destroyed) { + /* + * We have closed all wsi that were bound to this vhost + * by any pt: nothing can be servicing any wsi belonging + * to it any more. + * + * Finalize the vh destruction + */ + lws_vhost_destroy2(wsi->vhost); + } + wsi->vhost = NULL; + } +} + void __lws_free_wsi(struct lws *wsi) { @@ -129,6 +163,8 @@ __lws_free_wsi(struct lws *wsi) if (wsi->context->event_loop_ops->destroy_wsi) wsi->context->event_loop_ops->destroy_wsi(wsi); + lws_vhost_unbind_wsi(wsi); + wsi->context->count_wsi_allocated--; lwsl_debug("%s: %p, remaining wsi %d\n", __func__, wsi, wsi->context->count_wsi_allocated); @@ -1372,7 +1408,7 @@ lws_callback_vhost_protocols_vhost(struct lws_vhost *vh, int reason, void *in, struct lws *wsi = lws_zalloc(sizeof(*wsi), "fake wsi"); wsi->context = vh->context; - wsi->vhost = vh; + lws_vhost_bind_wsi(vh, wsi); for (n = 0; n < wsi->vhost->count_protocols; n++) { wsi->protocol = &vh->protocols[n]; @@ -1652,7 +1688,7 @@ lws_broadcast(struct lws_context *context, int reason, void *in, size_t len) while (v) { const struct lws_protocols *p = v->protocols; - wsi.vhost = v; + wsi.vhost = v; /* not a real bound wsi */ for (n = 0; n < v->count_protocols; n++) { wsi.protocol = p; diff --git a/lib/core/private.h b/lib/core/private.h index f9353d9c7..34ec34691 100644 --- a/lib/core/private.h +++ b/lib/core/private.h @@ -647,6 +647,7 @@ struct lws_vhost { #endif #if LWS_MAX_SMP > 1 pthread_mutex_t lock; + char close_flow_vs_tsi[LWS_MAX_SMP]; #endif #if defined(LWS_ROLE_H2) @@ -675,6 +676,9 @@ struct lws_vhost { const char *name; const char *iface; + void (*finalize)(struct lws_vhost *vh, void *arg); + void *finalize_arg; + #if !defined(LWS_WITH_ESP32) && !defined(OPTEE_TA) && !defined(WIN32) int bind_iface; #endif @@ -708,6 +712,8 @@ struct lws_vhost { int keepalive_timeout; int timeout_secs_ah_idle; + int count_bound_wsi; + #ifdef LWS_WITH_ACCESS_LOG int log_fd; #endif @@ -719,6 +725,13 @@ struct lws_vhost { unsigned char raw_protocol_index; }; +void +lws_vhost_bind_wsi(struct lws_vhost *vh, struct lws *wsi); +void +lws_vhost_unbind_wsi(struct lws *wsi); +void +lws_vhost_destroy2(struct lws_vhost *vh); + struct lws_deferred_free { struct lws_deferred_free *next; @@ -901,7 +914,7 @@ struct lws_context { }; int -lws_check_deferred_free(struct lws_context *context, int force); +lws_check_deferred_free(struct lws_context *context, int tsi, int force); #define lws_get_context_protocol(ctx, x) ctx->vhost_list->protocols[x] #define lws_get_vh_protocol(vh, x) vh->protocols[x] diff --git a/lib/core/service.c b/lib/core/service.c index 7ecc769e0..5cde4d2e8 100644 --- a/lib/core/service.c +++ b/lib/core/service.c @@ -602,7 +602,7 @@ lws_service_periodic_checks(struct lws_context *context, #endif lws_plat_service_periodic(context); - lws_check_deferred_free(context, 0); + lws_check_deferred_free(context, tsi, 0); #if defined(LWS_WITH_PEER_LIMITS) lws_peer_cull_peer_wait_list(context); @@ -752,7 +752,7 @@ lws_service_periodic_checks(struct lws_context *context, if (!wsi) wsi = lws_zalloc(sizeof(*wsi), "cbwsi"); wsi->context = context; - wsi->vhost = v; + wsi->vhost = v; /* not a real bound wsi */ wsi->protocol = q->protocol; lwsl_debug("timed cb: vh %s, protocol %s, reason %d\n", v->name, q->protocol->name, q->reason); q->protocol->callback(wsi, q->reason, NULL, NULL, 0); diff --git a/lib/libwebsockets.h b/lib/libwebsockets.h index 475ef41e5..f17992870 100644 --- a/lib/libwebsockets.h +++ b/lib/libwebsockets.h @@ -2978,6 +2978,21 @@ struct lws_context_creation_info { * native event library signal handle, eg uv_signal_t * * for libuv. */ + struct lws_context **pcontext; + /**< CONTEXT: if non-NULL, at the end of context destroy processing, + * the pointer pointed to by pcontext is written with NULL. You can + * use this to let foreign event loops know that lws context destruction + * is fully completed. + */ + void (*finalize)(struct lws_vhost *vh, void *arg); + /**< VHOST: NULL, or pointer to function that will be called back + * when the vhost is just about to be freed. The arg parameter + * will be set to whatever finalize_arg is below. + */ + void *finalize_arg; + /**< VHOST: opaque pointer lws ignores but passes to the finalize + * callback. If you don't care, leave it NULL. + */ /* Add new things just above here ---^ * This is part of the ABI, don't needlessly break compatibility @@ -2986,12 +3001,6 @@ struct lws_context_creation_info { * members added above will see 0 (default) even if the app * was not built against the newer headers. */ - struct lws_context **pcontext; - /**< CONTEXT: if non-NULL, at the end of context destroy processing, - * the pointer pointed to by pcontext is written with NULL. You can - * use this to let foreign event loops know that lws context destruction - * is fully completed. - */ void *_unused[4]; /**< dummy */ }; @@ -3137,7 +3146,7 @@ lws_create_vhost(struct lws_context *context, /** * lws_vhost_destroy() - Destroy a vhost (virtual server context) * - * \param vh: pointer to result of lws_create_vhost() + * \param vh: pointer to result of lws_create_vhost() * * This function destroys a vhost. Normally, if you just want to exit, * then lws_destroy_context() will take care of everything. If you want @@ -3146,6 +3155,11 @@ lws_create_vhost(struct lws_context *context, * * If the vhost has a listen sockets shared by other vhosts, it will be given * to one of the vhosts sharing it rather than closed. + * + * The vhost close is staged according to the needs of the event loop, and if + * there are multiple service threads. At the point the vhost itself if + * about to be freed, if you provided a finalize callback and optional arg at + * vhost creation time, it will be called just before the vhost is freed. */ LWS_VISIBLE LWS_EXTERN void lws_vhost_destroy(struct lws_vhost *vh); diff --git a/lib/roles/cgi/cgi-server.c b/lib/roles/cgi/cgi-server.c index d607e1767..7f3e5b64e 100644 --- a/lib/roles/cgi/cgi-server.c +++ b/lib/roles/cgi/cgi-server.c @@ -147,7 +147,7 @@ lws_cgi(struct lws *wsi, const char * const *exec_array, int script_uri_path_len if (!cgi->stdwsi[n]) goto bail2; cgi->stdwsi[n]->cgi_channel = n; - cgi->stdwsi[n]->vhost = wsi->vhost; + lws_vhost_bind_wsi(wsi->vhost, cgi->stdwsi[n]); lwsl_debug("%s: cgi %p: pipe fd %d -> fd %d / %d\n", __func__, cgi->stdwsi[n], n, cgi->pipe_fds[n][!!(n == 0)], diff --git a/lib/roles/h2/http2.c b/lib/roles/h2/http2.c index d630872fb..4e71adc45 100644 --- a/lib/roles/h2/http2.c +++ b/lib/roles/h2/http2.c @@ -222,6 +222,7 @@ bail1: if (wsi->user_space) lws_free_set_NULL(wsi->user_space); vh->protocols[0].callback(wsi, LWS_CALLBACK_WSI_DESTROY, NULL, NULL, 0); + lws_vhost_unbind_wsi(wsi); lws_free(wsi); return NULL; @@ -1071,7 +1072,7 @@ lws_h2_parse_frame_header(struct lws *wsi) pps = lws_h2_new_pps(LWS_H2_PPS_UPDATE_WINDOW); if (!pps) - return 1; + goto cleanup_wsi; pps->u.update_window.sid = h2n->sid; pps->u.update_window.credit = 4 * 65536; h2n->swsi->h2.peer_tx_cr_est += pps->u.update_window.credit; @@ -1079,7 +1080,7 @@ lws_h2_parse_frame_header(struct lws *wsi) pps = lws_h2_new_pps(LWS_H2_PPS_UPDATE_WINDOW); if (!pps) - return 1; + goto cleanup_wsi; pps->u.update_window.sid = 0; pps->u.update_window.credit = 4 * 65536; wsi->h2.peer_tx_cr_est += pps->u.update_window.credit; @@ -1135,6 +1136,10 @@ update_end_headers: lwsl_debug("END_HEADERS %d\n", h2n->swsi->h2.END_HEADERS); break; +cleanup_wsi: + + return 1; + case LWS_H2_FRAME_TYPE_WINDOW_UPDATE: if (h2n->length != 4) { lws_h2_goaway(wsi, H2_ERR_FRAME_SIZE_ERROR, diff --git a/lib/roles/http/server/server.c b/lib/roles/http/server/server.c index 07e925f4a..3b3629873 100644 --- a/lib/roles/http/server/server.c +++ b/lib/roles/http/server/server.c @@ -237,7 +237,7 @@ done_list: lws_role_transition(wsi, 0, LRS_UNCONNECTED, &role_ops_listen); wsi->protocol = vhost->protocols; wsi->tsi = m; - wsi->vhost = vhost; + lws_vhost_bind_wsi(vhost, wsi); wsi->listener = 1; if (wsi->context->event_loop_ops->init_vhost_listen_wsi) @@ -1453,7 +1453,7 @@ raw_transition: lws_hdr_simple_ptr(wsi, WSI_TOKEN_HOST)); if (vhost) - wsi->vhost = vhost; + lws_vhost_bind_wsi(vhost, wsi); } else lwsl_info("no host\n"); diff --git a/lib/roles/listen/ops-listen.c b/lib/roles/listen/ops-listen.c index 2aaee329d..4387cd2d9 100644 --- a/lib/roles/listen/ops-listen.c +++ b/lib/roles/listen/ops-listen.c @@ -32,6 +32,11 @@ rops_handle_POLLIN_listen(struct lws_context_per_thread *pt, struct lws *wsi, struct sockaddr_storage cli_addr; socklen_t clilen; + /* if our vhost is going down, ignore it */ + + if (wsi->vhost->being_destroyed) + return LWS_HPI_RET_HANDLED; + /* pollin means a client has connected to us then * * pollout is a hack on esp32 for background accepts signalling diff --git a/lib/tls/tls-client.c b/lib/tls/tls-client.c index 70eb6f607..8a5012101 100644 --- a/lib/tls/tls-client.c +++ b/lib/tls/tls-client.c @@ -139,7 +139,7 @@ int lws_context_init_client_ssl(const struct lws_context_creation_info *info, * lws_get_context() in the callback */ memset(&wsi, 0, sizeof(wsi)); - wsi.vhost = vhost; + wsi.vhost = vhost; /* not a real bound wsi */ wsi.context = vhost->context; vhost->protocols[0].callback(&wsi, diff --git a/lib/tls/tls-server.c b/lib/tls/tls-server.c index 440e79066..1f727cf79 100644 --- a/lib/tls/tls-server.c +++ b/lib/tls/tls-server.c @@ -139,7 +139,7 @@ lws_context_init_server_ssl(const struct lws_context_creation_info *info, * lws_get_context() in the callback */ memset(&wsi, 0, sizeof(wsi)); - wsi.vhost = vhost; + wsi.vhost = vhost; /* not a real bound wsi */ wsi.context = context; /* @@ -354,7 +354,7 @@ accepted: if (!vh->being_destroyed && wsi->tls.ssl && vh->tls.ssl_ctx == lws_tls_ctx_from_wsi(wsi)) { lwsl_info("setting wsi to vh %s\n", vh->name); - wsi->vhost = vh; + lws_vhost_bind_wsi(vh, wsi); break; } vh = vh->vhost_next; diff --git a/lib/tls/tls.c b/lib/tls/tls.c index 92b7c5593..8cf39ab17 100644 --- a/lib/tls/tls.c +++ b/lib/tls/tls.c @@ -446,7 +446,7 @@ lws_tls_cert_updated(struct lws_context *context, const char *certpath, wsi.context = context; lws_start_foreach_ll(struct lws_vhost *, v, context->vhost_list) { - wsi.vhost = v; + wsi.vhost = v; /* not a real bound wsi */ if (v->tls.alloc_cert_path && v->tls.key_path && !strcmp(v->tls.alloc_cert_path, certpath) && !strcmp(v->tls.key_path, keypath)) {