Mirrors/libwebsockets
1
0
Fork 0
mirror of https://github.com/warmcat/libwebsockets.git synced 2025-03-09 00:00:04 +01:00
Code Issues Releases Wiki Activity

vhost_destroy: use vhost wsi reference counting to trigger destroy

Browse source 
This changes the vhost destroy flow to only hand off the listen
socket if another vhost sharing it, and mark the vhost as
being_destroyed.

Each tsi calls lws_check_deferred_free() once a second, if it sees
any vhost being_destroyed there, it closes all wsi on its tsi on
the same vhost, one time.

As the wsi on the vhost complete close (ie, after libuv async close
if on libuv event loop), they decrement a reference count for all
wsi open on the vhost.  The tsi who closes the last one then
completes the destroy flow for the vhost itself... it's random
which tsi completes the vhost destroy but since there are no
wsi left on the vhost, and it holds the context lock, nothing
can conflict.

The advantage of this is that owning tsi do the close for wsi
that are bound to the vhost under destruction, at a time when
they are guaranteed to be idle for service, and they do it with
both vhost and context locks owned, so no other service thread
can conflict for stuff protected by those either.

For the situation the user code may have allocations attached to
the vhost, this adds args to lws_vhost_destroy() to allow destroying
the user allocations just before the vhost is freed.
This commit is contained in:
Andy Green 2018-06-16 09:31:07 +08:00
parent 2935d7d32f
commit ac3bd36c60
14 changed files with 209 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
lib/core/adopt.c
View file

@ -64,7 +64,7 @@ lws_create_new_server_wsi(struct lws_vhost *vhost, int fixed_tsi)
lwsl_debug("new wsi %p joining vhost %s, tsi %d\n", new_wsi,
vhost->name, new_wsi->tsi);
new_wsi->vhost = vhost;
lws_vhost_bind_wsi(vhost, new_wsi);
new_wsi->context = vhost->context;
new_wsi->pending_timeout = NO_PENDING_TIMEOUT;
new_wsi->rxflow_change_to = LWS_RXFLOW_ALLOW;
@ -241,8 +241,10 @@ bail:
parent->child_list = new_wsi->sibling_list;
if (new_wsi->user_space)
lws_free(new_wsi->user_space);
lws_vhost_unbind_wsi(new_wsi);
lws_free(new_wsi);
compatible_close(fd.sockfd);
compatible_close(fd.sockfd);
return NULL;
}

2
lib/core/connect.c
View file

@ -97,6 +97,8 @@ lws_client_connect_via_info(const struct lws_client_connect_info *i)
goto bail;
}
lws_vhost_bind_wsi(wsi->vhost, wsi);
wsi->protocol = &wsi->vhost->protocols[0];
wsi->client_pipeline = !!(i->ssl_connection & LCCSCF_PIPELINE);

171
lib/core/context.c
View file

@ -698,9 +698,6 @@ static const struct lws_protocols protocols_dummy[] = {
#undef LWS_HAVE_GETENV
#endif
static void
lws_vhost_destroy2(struct lws_vhost *vh);
LWS_VISIBLE struct lws_vhost *
lws_create_vhost(struct lws_context *context,
const struct lws_context_creation_info *info)
@ -759,6 +756,8 @@ lws_create_vhost(struct lws_context *context,
vh->pvo = info->pvo;
vh->headers = info->headers;
vh->user = info->user;
vh->finalize = info->finalize;
vh->finalize_arg = info->finalize_arg;
LWS_FOR_EVERY_AVAILABLE_ROLE_START(ar)
if (ar->init_vhost)
@ -1025,8 +1024,7 @@ lws_create_vhost(struct lws_context *context,
return vh;
bail1:
lws_vhost_destroy(vh);
lws_vhost_destroy2(vh);
lws_vhost_destroy(vh, NULL, NULL);
return NULL;
@ -1614,24 +1612,27 @@ lws_context_is_deprecated(struct lws_context *context)
void
lws_vhost_destroy1(struct lws_vhost *vh)
{
const struct lws_protocols *protocol = NULL;
struct lws_context_per_thread *pt;
int n, m = vh->context->count_threads;
struct lws_context *context = vh->context;
struct lws wsi;
lwsl_info("%s\n", __func__);
if (vh->being_destroyed)
return;
lws_vhost_lock(vh); /* -------------- vh { */
vh->being_destroyed = 1;
/*
* PHASE 1: take down or reassign any listen wsi
*
* Are there other vhosts that are piggybacking on our listen socket?
* If so we need to hand the listen socket off to one of the others
* so it will remain open. If not, leave it attached to the closing
* vhost and it will get closed.
* so it will remain open.
*
* If not, leave it attached to the closing vhost, the vh being marked
* being_destroyed will defeat any service and it will get closed in
* later phases.
*/
if (vh->lserv_wsi)
@ -1652,9 +1653,11 @@ lws_vhost_destroy1(struct lws_vhost *vh)
*/
assert(v->lserv_wsi == NULL);
v->lserv_wsi = vh->lserv_wsi;
vh->lserv_wsi = NULL;
if (v->lserv_wsi)
v->lserv_wsi->vhost = v;
if (v->lserv_wsi) {
lws_vhost_unbind_wsi(vh->lserv_wsi);
lws_vhost_bind_wsi(v, v->lserv_wsi);
}
lwsl_notice("%s: listen skt from %s to %s\n",
__func__, vh->name, v->name);
@ -1662,29 +1665,25 @@ lws_vhost_destroy1(struct lws_vhost *vh)
}
} lws_end_foreach_ll(v, vhost_next);
lws_vhost_unlock(vh); /* } vh -------------- */
/*
* Forcibly close every wsi assoicated with this vhost. That will
* include the listen socket if it is still associated with the closing
* vhost.
* lws_check_deferred_free() will notice there is a vhost that is
* marked for destruction during the next 1s, for all tsi.
*
* It will start closing all wsi on this vhost. When the last wsi
* is closed, it will trigger lws_vhost_destroy2()
*/
}
while (m--) {
pt = &context->pt[m];
for (n = 0; (unsigned int)n < context->pt[m].fds_count; n++) {
struct lws *wsi = wsi_from_fd(context, pt->fds[n].fd);
if (!wsi)
continue;
if (wsi->vhost != vh)
continue;
lws_close_free_wsi(wsi,
LWS_CLOSE_STATUS_NOSTATUS_CONTEXT_DESTROY,
"vh destroy"
/* no protocol close */);
n--;
}
}
void
lws_vhost_destroy2(struct lws_vhost *vh)
{
const struct lws_protocols *protocol = NULL;
struct lws_context *context = vh->context;
struct lws_deferred_free *df;
struct lws wsi;
int n;
/*
* destroy any pending timed events
@ -1699,7 +1698,7 @@ lws_vhost_destroy1(struct lws_vhost *vh)
memset(&wsi, 0, sizeof(wsi));
wsi.context = vh->context;
wsi.vhost = vh;
wsi.vhost = vh; /* not a real bound wsi */
protocol = vh->protocols;
if (protocol && vh->created_vhost_protocols) {
n = 0;
@ -1727,15 +1726,6 @@ lws_vhost_destroy1(struct lws_vhost *vh)
vh->vhost_next = vh->context->vhost_pending_destruction_list;
vh->context->vhost_pending_destruction_list = vh;
}
static void
lws_vhost_destroy2(struct lws_vhost *vh)
{
const struct lws_protocols *protocol = NULL;
struct lws_context *context = vh->context;
struct lws_deferred_free *df;
int n;
lwsl_info("%s: %p\n", __func__, vh);
@ -1819,31 +1809,74 @@ lws_vhost_destroy2(struct lws_vhost *vh)
* they do not refer to the vhost. So it's safe to free.
*/
if (vh->finalize)
vh->finalize(vh, vh->finalize_arg);
lwsl_info(" %s: Freeing vhost %p\n", __func__, vh);
memset(vh, 0, sizeof(*vh));
lws_free(vh);
}
int
lws_check_deferred_free(struct lws_context *context, int force)
{
struct lws_deferred_free *df;
time_t now = lws_now_secs();
/*
* each service thread calls this once a second or so
*/
lws_start_foreach_llp(struct lws_deferred_free **, pdf,
context->deferred_free_list) {
if (force ||
lws_compare_time_t(context, now, (*pdf)->deadline) > 5) {
df = *pdf;
*pdf = df->next;
/* finalize vh destruction */
lwsl_notice("deferred vh %p destroy\n", df->payload);
lws_vhost_destroy2(df->payload);
lws_free(df);
continue; /* after deletion we already point to next */
int
lws_check_deferred_free(struct lws_context *context, int tsi, int force)
{
struct lws_context_per_thread *pt;
int n;
/*
* If we see a vhost is being destroyed, forcibly close every wsi on
* this tsi associated with this vhost. That will include the listen
* socket if it is still associated with the closing vhost.
*
* For SMP, we do this once per tsi per destroyed vhost. The reference
* counting on the vhost as the bound wsi close will notice that there
* are no bound wsi left, that vhost destruction can complete,
* and perform it. It doesn't matter which service thread does that
* because there is nothing left using the vhost to conflict.
*/
lws_context_lock(context); /* ------------------- context { */
lws_start_foreach_ll(struct lws_vhost *, v, context->vhost_list) {
if (v->being_destroyed
#if LWS_MAX_SMP > 1
&& !v->close_flow_vs_tsi[tsi]
#endif
) {
pt = &context->pt[tsi];
lws_pt_lock(pt, "vhost removal"); /* -------------- pt { */
#if LWS_MAX_SMP > 1
v->close_flow_vs_tsi[tsi] = 1;
#endif
for (n = 0; (unsigned int)n < pt->fds_count; n++) {
struct lws *wsi = wsi_from_fd(context, pt->fds[n].fd);
if (!wsi)
continue;
if (wsi->vhost != v)
continue;
__lws_close_free_wsi(wsi,
LWS_CLOSE_STATUS_NOSTATUS_CONTEXT_DESTROY,
"vh destroy"
/* no protocol close */);
n--;
}
lws_pt_unlock(pt); /* } pt -------------- */
}
} lws_end_foreach_llp(pdf, next);
} lws_end_foreach_ll(v, vhost_next);
lws_context_unlock(context); /* } context ------------------- */
return 0;
}
@ -1858,6 +1891,20 @@ lws_vhost_destroy(struct lws_vhost *vh)
lws_vhost_destroy1(vh);
if (!vh->count_bound_wsi) {
/*
* After listen handoff, there are already no wsi bound to this
* vhost by any pt: nothing can be servicing any wsi belonging
* to it any more.
*
* Finalize the vh destruction immediately
*/
lws_vhost_destroy2(vh);
lws_free(df);
return;
}
/* part 2 is deferred to allow all the handle closes to complete */
df->next = vh->context->deferred_free_list;
@ -1980,7 +2027,7 @@ lws_context_destroy2(struct lws_context *context)
if (context->external_baggage_free_on_destroy)
free(context->external_baggage_free_on_destroy);
lws_check_deferred_free(context, 1);
lws_check_deferred_free(context, 0, 1);
#if LWS_MAX_SMP > 1
pthread_mutex_destroy(&context->lock);

40
lib/core/libwebsockets.c
View file

@ -88,6 +88,40 @@ signed char char_to_hex(const char c)
return -1;
}
void
lws_vhost_bind_wsi(struct lws_vhost *vh, struct lws *wsi)
{
wsi->vhost = vh;
vh->count_bound_wsi++;
lwsl_info("%s: vh %s: count_bound_wsi %d\n",
__func__, vh->name, vh->count_bound_wsi);
assert(wsi->vhost->count_bound_wsi > 0);
}
void
lws_vhost_unbind_wsi(struct lws *wsi)
{
if (wsi->vhost) {
assert(wsi->vhost->count_bound_wsi > 0);
wsi->vhost->count_bound_wsi--;
lwsl_info("%s: vh %s: count_bound_wsi %d\n", __func__,
wsi->vhost->name, wsi->vhost->count_bound_wsi);
if (!wsi->vhost->count_bound_wsi &&
wsi->vhost->being_destroyed) {
/*
* We have closed all wsi that were bound to this vhost
* by any pt: nothing can be servicing any wsi belonging
* to it any more.
*
* Finalize the vh destruction
*/
lws_vhost_destroy2(wsi->vhost);
}
wsi->vhost = NULL;
}
}
void
__lws_free_wsi(struct lws *wsi)
{
@ -129,6 +163,8 @@ __lws_free_wsi(struct lws *wsi)
if (wsi->context->event_loop_ops->destroy_wsi)
wsi->context->event_loop_ops->destroy_wsi(wsi);
lws_vhost_unbind_wsi(wsi);
wsi->context->count_wsi_allocated--;
lwsl_debug("%s: %p, remaining wsi %d\n", __func__, wsi,
wsi->context->count_wsi_allocated);
@ -1372,7 +1408,7 @@ lws_callback_vhost_protocols_vhost(struct lws_vhost *vh, int reason, void *in,
struct lws *wsi = lws_zalloc(sizeof(*wsi), "fake wsi");
wsi->context = vh->context;
wsi->vhost = vh;
lws_vhost_bind_wsi(vh, wsi);
for (n = 0; n < wsi->vhost->count_protocols; n++) {
wsi->protocol = &vh->protocols[n];
@ -1652,7 +1688,7 @@ lws_broadcast(struct lws_context *context, int reason, void *in, size_t len)
while (v) {
const struct lws_protocols *p = v->protocols;
wsi.vhost = v;
wsi.vhost = v; /* not a real bound wsi */
for (n = 0; n < v->count_protocols; n++) {
wsi.protocol = p;

15
lib/core/private.h
View file

@ -647,6 +647,7 @@ struct lws_vhost {
#endif
#if LWS_MAX_SMP > 1
pthread_mutex_t lock;
char close_flow_vs_tsi[LWS_MAX_SMP];
#endif
#if defined(LWS_ROLE_H2)
@ -675,6 +676,9 @@ struct lws_vhost {
const char *name;
const char *iface;
void (*finalize)(struct lws_vhost *vh, void *arg);
void *finalize_arg;
#if !defined(LWS_WITH_ESP32) && !defined(OPTEE_TA) && !defined(WIN32)
int bind_iface;
#endif
@ -708,6 +712,8 @@ struct lws_vhost {
int keepalive_timeout;
int timeout_secs_ah_idle;
int count_bound_wsi;
#ifdef LWS_WITH_ACCESS_LOG
int log_fd;
#endif
@ -719,6 +725,13 @@ struct lws_vhost {
unsigned char raw_protocol_index;
};
void
lws_vhost_bind_wsi(struct lws_vhost *vh, struct lws *wsi);
void
lws_vhost_unbind_wsi(struct lws *wsi);
void
lws_vhost_destroy2(struct lws_vhost *vh);
struct lws_deferred_free
{
struct lws_deferred_free *next;
@ -901,7 +914,7 @@ struct lws_context {
};
int
lws_check_deferred_free(struct lws_context *context, int force);
lws_check_deferred_free(struct lws_context *context, int tsi, int force);
#define lws_get_context_protocol(ctx, x) ctx->vhost_list->protocols[x]
#define lws_get_vh_protocol(vh, x) vh->protocols[x]

4
lib/core/service.c
View file

@ -602,7 +602,7 @@ lws_service_periodic_checks(struct lws_context *context,
#endif
lws_plat_service_periodic(context);
lws_check_deferred_free(context, 0);
lws_check_deferred_free(context, tsi, 0);
#if defined(LWS_WITH_PEER_LIMITS)
lws_peer_cull_peer_wait_list(context);
@ -752,7 +752,7 @@ lws_service_periodic_checks(struct lws_context *context,
if (!wsi)
wsi = lws_zalloc(sizeof(*wsi), "cbwsi");
wsi->context = context;
wsi->vhost = v;
wsi->vhost = v; /* not a real bound wsi */
wsi->protocol = q->protocol;
lwsl_debug("timed cb: vh %s, protocol %s, reason %d\n", v->name, q->protocol->name, q->reason);
q->protocol->callback(wsi, q->reason, NULL, NULL, 0);

28
lib/libwebsockets.h
View file

@ -2978,6 +2978,21 @@ struct lws_context_creation_info {
* native event library signal handle, eg uv_signal_t *
* for libuv.
*/
struct lws_context **pcontext;
/**< CONTEXT: if non-NULL, at the end of context destroy processing,
* the pointer pointed to by pcontext is written with NULL. You can
* use this to let foreign event loops know that lws context destruction
* is fully completed.
*/
void (*finalize)(struct lws_vhost *vh, void *arg);
/**< VHOST: NULL, or pointer to function that will be called back
* when the vhost is just about to be freed. The arg parameter
* will be set to whatever finalize_arg is below.
*/
void *finalize_arg;
/**< VHOST: opaque pointer lws ignores but passes to the finalize
* callback. If you don't care, leave it NULL.
*/
/* Add new things just above here ---^
* This is part of the ABI, don't needlessly break compatibility
@ -2986,12 +3001,6 @@ struct lws_context_creation_info {
* members added above will see 0 (default) even if the app
* was not built against the newer headers.
*/
struct lws_context **pcontext;
/**< CONTEXT: if non-NULL, at the end of context destroy processing,
* the pointer pointed to by pcontext is written with NULL. You can
* use this to let foreign event loops know that lws context destruction
* is fully completed.
*/
void *_unused[4]; /**< dummy */
};
@ -3137,7 +3146,7 @@ lws_create_vhost(struct lws_context *context,
/**
* lws_vhost_destroy() - Destroy a vhost (virtual server context)
*
* \param vh: pointer to result of lws_create_vhost()
* \param vh: pointer to result of lws_create_vhost()
*
* This function destroys a vhost. Normally, if you just want to exit,
* then lws_destroy_context() will take care of everything. If you want
@ -3146,6 +3155,11 @@ lws_create_vhost(struct lws_context *context,
*
* If the vhost has a listen sockets shared by other vhosts, it will be given
* to one of the vhosts sharing it rather than closed.
*
* The vhost close is staged according to the needs of the event loop, and if
* there are multiple service threads. At the point the vhost itself if
* about to be freed, if you provided a finalize callback and optional arg at
* vhost creation time, it will be called just before the vhost is freed.
*/
LWS_VISIBLE LWS_EXTERN void
lws_vhost_destroy(struct lws_vhost *vh);

2
lib/roles/cgi/cgi-server.c
View file

@ -147,7 +147,7 @@ lws_cgi(struct lws *wsi, const char * const *exec_array, int script_uri_path_len
if (!cgi->stdwsi[n])
goto bail2;
cgi->stdwsi[n]->cgi_channel = n;
cgi->stdwsi[n]->vhost = wsi->vhost;
lws_vhost_bind_wsi(wsi->vhost, cgi->stdwsi[n]);
lwsl_debug("%s: cgi %p: pipe fd %d -> fd %d / %d\n", __func__,
cgi->stdwsi[n], n, cgi->pipe_fds[n][!!(n == 0)],

9
lib/roles/h2/http2.c
View file

@ -222,6 +222,7 @@ bail1:
if (wsi->user_space)
lws_free_set_NULL(wsi->user_space);
vh->protocols[0].callback(wsi, LWS_CALLBACK_WSI_DESTROY, NULL, NULL, 0);
lws_vhost_unbind_wsi(wsi);
lws_free(wsi);
return NULL;
@ -1071,7 +1072,7 @@ lws_h2_parse_frame_header(struct lws *wsi)
pps = lws_h2_new_pps(LWS_H2_PPS_UPDATE_WINDOW);
if (!pps)
return 1;
goto cleanup_wsi;
pps->u.update_window.sid = h2n->sid;
pps->u.update_window.credit = 4 * 65536;
h2n->swsi->h2.peer_tx_cr_est += pps->u.update_window.credit;
@ -1079,7 +1080,7 @@ lws_h2_parse_frame_header(struct lws *wsi)
pps = lws_h2_new_pps(LWS_H2_PPS_UPDATE_WINDOW);
if (!pps)
return 1;
goto cleanup_wsi;
pps->u.update_window.sid = 0;
pps->u.update_window.credit = 4 * 65536;
wsi->h2.peer_tx_cr_est += pps->u.update_window.credit;
@ -1135,6 +1136,10 @@ update_end_headers:
lwsl_debug("END_HEADERS %d\n", h2n->swsi->h2.END_HEADERS);
break;
cleanup_wsi:
return 1;
case LWS_H2_FRAME_TYPE_WINDOW_UPDATE:
if (h2n->length != 4) {
lws_h2_goaway(wsi, H2_ERR_FRAME_SIZE_ERROR,

4
lib/roles/http/server/server.c
View file

@ -237,7 +237,7 @@ done_list:
lws_role_transition(wsi, 0, LRS_UNCONNECTED, &role_ops_listen);
wsi->protocol = vhost->protocols;
wsi->tsi = m;
wsi->vhost = vhost;
lws_vhost_bind_wsi(vhost, wsi);
wsi->listener = 1;
if (wsi->context->event_loop_ops->init_vhost_listen_wsi)
@ -1453,7 +1453,7 @@ raw_transition:
lws_hdr_simple_ptr(wsi, WSI_TOKEN_HOST));
if (vhost)
wsi->vhost = vhost;
lws_vhost_bind_wsi(vhost, wsi);
} else
lwsl_info("no host\n");

5
lib/roles/listen/ops-listen.c
View file

@ -32,6 +32,11 @@ rops_handle_POLLIN_listen(struct lws_context_per_thread *pt, struct lws *wsi,
struct sockaddr_storage cli_addr;
socklen_t clilen;
/* if our vhost is going down, ignore it */
if (wsi->vhost->being_destroyed)
return LWS_HPI_RET_HANDLED;
/* pollin means a client has connected to us then
*
* pollout is a hack on esp32 for background accepts signalling

2
lib/tls/tls-client.c
View file

@ -139,7 +139,7 @@ int lws_context_init_client_ssl(const struct lws_context_creation_info *info,
* lws_get_context() in the callback
*/
memset(&wsi, 0, sizeof(wsi));
wsi.vhost = vhost;
wsi.vhost = vhost; /* not a real bound wsi */
wsi.context = vhost->context;
vhost->protocols[0].callback(&wsi,

4
lib/tls/tls-server.c
View file

@ -139,7 +139,7 @@ lws_context_init_server_ssl(const struct lws_context_creation_info *info,
* lws_get_context() in the callback
*/
memset(&wsi, 0, sizeof(wsi));
wsi.vhost = vhost;
wsi.vhost = vhost; /* not a real bound wsi */
wsi.context = context;
/*
@ -354,7 +354,7 @@ accepted:
if (!vh->being_destroyed && wsi->tls.ssl &&
vh->tls.ssl_ctx == lws_tls_ctx_from_wsi(wsi)) {
lwsl_info("setting wsi to vh %s\n", vh->name);
wsi->vhost = vh;
lws_vhost_bind_wsi(vh, wsi);
break;
}
vh = vh->vhost_next;

2
lib/tls/tls.c
View file

@ -446,7 +446,7 @@ lws_tls_cert_updated(struct lws_context *context, const char *certpath,
wsi.context = context;
lws_start_foreach_ll(struct lws_vhost *, v, context->vhost_list) {
wsi.vhost = v;
wsi.vhost = v; /* not a real bound wsi */
if (v->tls.alloc_cert_path && v->tls.key_path &&
!strcmp(v->tls.alloc_cert_path, certpath) &&
!strcmp(v->tls.key_path, keypath)) {