diff --git a/lib/context.c b/lib/context.c index 3f452162..037f4431 100644 --- a/lib/context.c +++ b/lib/context.c @@ -1436,7 +1436,7 @@ LWS_VISIBLE void lws_context_destroy2(struct lws_context *context); -static void +void lws_vhost_destroy1(struct lws_vhost *vh) { const struct lws_protocols *protocol = NULL; @@ -1700,8 +1700,8 @@ LWS_VISIBLE void lws_context_destroy(struct lws_context *context) { volatile struct lws_foreign_thread_pollfd *ftp, *next; - struct lws_context_per_thread *pt; volatile struct lws_context_per_thread *vpt; + struct lws_context_per_thread *pt; struct lws_vhost *vh = NULL; struct lws wsi; int n, m; @@ -1798,6 +1798,19 @@ lws_context_destroy(struct lws_context *context) } lws_plat_context_early_destroy(context); +#if defined(LWS_WITH_LIBUV) + if (LWS_LIBUV_ENABLED(context)) + for (n = 0; n < context->count_threads; n++) { + pt = &context->pt[n]; + if (!pt->ev_loop_foreign) { +#if UV_VERSION_MAJOR > 0 + uv_loop_close(pt->io_loop_uv); +#endif + lws_free_set_NULL(pt->io_loop_uv); + } + } +#endif + if (context->pt[0].fds) lws_free_set_NULL(context->pt[0].fds); diff --git a/lib/event-libs/libuv.c b/lib/event-libs/libuv.c index 77138e55..f84ed458 100644 --- a/lib/event-libs/libuv.c +++ b/lib/event-libs/libuv.c @@ -240,6 +240,8 @@ lws_uv_initloop(struct lws_context *context, uv_loop_t *loop, int tsi) pt->io_loop_uv = loop; uv_idle_init(loop, &pt->uv_idle); + LWS_UV_REFCOUNT_STATIC_HANDLE_NEW(&pt->uv_idle, context); + ns = ARRAY_SIZE(sigs); if (lws_check_opt(context->options, @@ -250,6 +252,8 @@ lws_uv_initloop(struct lws_context *context, uv_loop_t *loop, int tsi) assert(ns <= (int)ARRAY_SIZE(pt->signals)); for (n = 0; n < ns; n++) { uv_signal_init(loop, &pt->signals[n]); + LWS_UV_REFCOUNT_STATIC_HANDLE_NEW(&pt->signals[n], + context); pt->signals[n].data = pt->context; uv_signal_start(&pt->signals[n], context->lws_uv_sigint_cb, @@ -275,12 +279,14 @@ lws_uv_initloop(struct lws_context *context, uv_loop_t *loop, int tsi) vh = vh->vhost_next; } - if (first) { - uv_timer_init(pt->io_loop_uv, &pt->uv_timeout_watcher); - uv_timer_start(&pt->uv_timeout_watcher, lws_uv_timeout_cb, - 10, 1000); - uv_timer_init(pt->io_loop_uv, &pt->uv_hrtimer); - } + if (!first) + return status; + + uv_timer_init(pt->io_loop_uv, &pt->uv_timeout_watcher); + LWS_UV_REFCOUNT_STATIC_HANDLE_NEW(&pt->uv_timeout_watcher, context); + uv_timer_start(&pt->uv_timeout_watcher, lws_uv_timeout_cb, 10, 1000); + uv_timer_init(pt->io_loop_uv, &pt->uv_hrtimer); + LWS_UV_REFCOUNT_STATIC_HANDLE_NEW(&pt->uv_hrtimer, context); return status; @@ -288,6 +294,73 @@ bail: return -1; } +/* + * Closing Phase 2: Close callback for a static UV asset + */ + +static void +lws_uv_close_cb_sa(uv_handle_t *handle) +{ + struct lws_context *context = + LWS_UV_REFCOUNT_STATIC_HANDLE_TO_CONTEXT(handle); + int n; + + lwsl_info("%s: sa left %d: dyn left: %d\n", __func__, + context->uv_count_static_asset_handles, + context->count_wsi_allocated); + + /* any static assets left? */ + + if (LWS_UV_REFCOUNT_STATIC_HANDLE_DESTROYED(handle) || + context->count_wsi_allocated) + return; + + /* + * That's it... all wsi were down, and now every + * static asset lws had a UV handle for is down. + * + * Stop the loop so we can get out of here. + */ + + for (n = 0; n < context->count_threads; n++) { + struct lws_context_per_thread *pt = &context->pt[n]; + + if (!pt->io_loop_uv || !LWS_LIBUV_ENABLED(context)) + continue; + + uv_stop(pt->io_loop_uv); + + /* + * we can't delete non-foreign loop here, because + * the uv_stop() hasn't got us out of the uv_run() + * yet. So we do it in context destroy. + */ + } +} + +/* + * These must be called by protocols that want to use libuv objects directly... + * + * .... when the libuv object is created... + */ + +LWS_VISIBLE void +lws_libuv_static_refcount_add(uv_handle_t *h, struct lws_context *context) +{ + LWS_UV_REFCOUNT_STATIC_HANDLE_NEW(h, context); +} + +/* + * ... and in the close callback when the object is closed. + */ + +LWS_VISIBLE void +lws_libuv_static_refcount_del(uv_handle_t *h) +{ + return lws_uv_close_cb_sa(h); +} + + static void lws_uv_close_cb(uv_handle_t *handle) { } @@ -308,7 +381,9 @@ void lws_libuv_destroyloop(struct lws_context *context, int tsi) { struct lws_context_per_thread *pt = &context->pt[tsi]; - int m, budget = 100, ns; + int m, /* budget = 100, */ ns; + + lwsl_info("%s: %d\n", __func__, tsi); if (!lws_check_opt(context->options, LWS_SERVER_OPTION_LIBUV)) return; @@ -316,6 +391,11 @@ lws_libuv_destroyloop(struct lws_context *context, int tsi) if (!pt->io_loop_uv) return; + if (pt->event_loop_destroy_processing_done) + return; + + pt->event_loop_destroy_processing_done = 1; + if (context->use_ev_sigint) { uv_signal_stop(&pt->w_sigint.uv_watcher); @@ -326,34 +406,17 @@ lws_libuv_destroyloop(struct lws_context *context, int tsi) for (m = 0; m < ns; m++) { uv_signal_stop(&pt->signals[m]); - uv_close((uv_handle_t *)&pt->signals[m], lws_uv_close_cb); + uv_close((uv_handle_t *)&pt->signals[m], lws_uv_close_cb_sa); } } uv_timer_stop(&pt->uv_timeout_watcher); - uv_close((uv_handle_t *)&pt->uv_timeout_watcher, lws_uv_close_cb); + uv_close((uv_handle_t *)&pt->uv_timeout_watcher, lws_uv_close_cb_sa); uv_timer_stop(&pt->uv_hrtimer); - uv_close((uv_handle_t *)&pt->uv_hrtimer, lws_uv_close_cb); + uv_close((uv_handle_t *)&pt->uv_hrtimer, lws_uv_close_cb_sa); uv_idle_stop(&pt->uv_idle); - uv_close((uv_handle_t *)&pt->uv_idle, lws_uv_close_cb); - - while (budget-- && uv_run(pt->io_loop_uv, UV_RUN_NOWAIT)) - ; - - if (pt->ev_loop_foreign) - return; - - uv_stop(pt->io_loop_uv); - uv_walk(pt->io_loop_uv, lws_uv_walk_cb, NULL); - while (uv_run(pt->io_loop_uv, UV_RUN_NOWAIT)) - ; -#if UV_VERSION_MAJOR > 0 - m = uv_loop_close(pt->io_loop_uv); - if (m == UV_EBUSY) - lwsl_err("%s: uv_loop_close: UV_EBUSY\n", __func__); -#endif - lws_free(pt->io_loop_uv); + uv_close((uv_handle_t *)&pt->uv_idle, lws_uv_close_cb_sa); } void @@ -456,15 +519,75 @@ lws_libuv_stop_without_kill(const struct lws_context *context, int tsi) uv_stop(context->pt[tsi].io_loop_uv); } -static void -lws_libuv_kill(const struct lws_context *context) -{ - int n; - for (n = 0; n < context->count_threads; n++) - if (context->pt[n].io_loop_uv && - LWS_LIBUV_ENABLED(context)) - uv_stop(context->pt[n].io_loop_uv); + +LWS_VISIBLE uv_loop_t * +lws_uv_getloop(struct lws_context *context, int tsi) +{ + if (context->pt[tsi].io_loop_uv && LWS_LIBUV_ENABLED(context)) + return context->pt[tsi].io_loop_uv; + + return NULL; +} + +static void +lws_libuv_closewsi(uv_handle_t* handle) +{ + struct lws *n = NULL, *wsi = (struct lws *)(((char *)handle) - + (char *)(&n->w_read.uv_watcher)); + struct lws_context *context = lws_get_context(wsi); + struct lws_context_per_thread *pt = &context->pt[(int)wsi->tsi]; + int lspd = 0, m; + + /* + * We get called back here for every wsi that closes + */ + + if (wsi->mode == LWSCM_SERVER_LISTENER && + wsi->context->deprecated) { + lspd = 1; + context->deprecation_pending_listen_close_count--; + if (!context->deprecation_pending_listen_close_count) + lspd = 2; + } + + lws_pt_lock(pt, __func__); + __lws_close_free_wsi_final(wsi); + lws_pt_unlock(pt); + + if (lspd == 2 && context->deprecation_cb) { + lwsl_notice("calling deprecation callback\n"); + context->deprecation_cb(); + } + + lwsl_info("%s: sa left %d: dyn left: %d\n", __func__, + context->uv_count_static_asset_handles, + context->count_wsi_allocated); + + /* + * eventually, we closed all the wsi... + */ + + if (context->requested_kill && !context->count_wsi_allocated) { + struct lws_vhost *vh = context->vhost_list; + + /* + * Start Closing Phase 2: close of static handles + */ + + lwsl_info("%s: all lws dynamic handles down, closing static\n", + __func__); + + for (m = 0; m < context->count_threads; m++) + lws_libuv_destroyloop(context, m); + + /* protocols may have initialized libuv objects */ + + while (vh) { + lws_vhost_destroy1(vh); + vh = vh->vhost_next; + } + } } /* @@ -487,6 +610,10 @@ lws_libuv_stop(struct lws_context *context) m = context->count_threads; context->being_destroyed = 1; + /* + * Phase 1: start the close of every dynamic uv handle + */ + while (m--) { pt = &context->pt[m]; @@ -503,59 +630,15 @@ lws_libuv_stop(struct lws_context *context) } lwsl_info("%s: started closing all wsi\n", __func__); - if (context->count_wsi_allocated == 0) - lws_libuv_kill(context); -} -LWS_VISIBLE uv_loop_t * -lws_uv_getloop(struct lws_context *context, int tsi) -{ - if (context->pt[tsi].io_loop_uv && LWS_LIBUV_ENABLED(context)) - return context->pt[tsi].io_loop_uv; - - return NULL; -} - -static void -lws_libuv_closewsi(uv_handle_t* handle) -{ - struct lws *n = NULL, *wsi = (struct lws *)(((char *)handle) - - (char *)(&n->w_read.uv_watcher)); - struct lws_context *context = lws_get_context(wsi); - struct lws_context_per_thread *pt = &context->pt[(int)wsi->tsi]; - int lspd = 0; - - if (wsi->mode == LWSCM_SERVER_LISTENER && - wsi->context->deprecated) { - lspd = 1; - context->deprecation_pending_listen_close_count--; - if (!context->deprecation_pending_listen_close_count) - lspd = 2; - } - - lws_pt_lock(pt, __func__); - __lws_close_free_wsi_final(wsi); - lws_pt_unlock(pt); - - if (lspd == 2 && context->deprecation_cb) { - lwsl_notice("calling deprecation callback\n"); - context->deprecation_cb(); - } - - if (context->requested_kill && context->count_wsi_allocated == 0) - lws_libuv_kill(context); + /* we cannot have completed... there are at least the cancel pipes */ } void lws_libuv_closehandle(struct lws *wsi) { - struct lws_context *context = lws_get_context(wsi); - /* required to defer actual deletion until libuv has processed it */ uv_close((uv_handle_t*)&wsi->w_read.uv_watcher, lws_libuv_closewsi); - - if (context->requested_kill && context->count_wsi_allocated == 0) - lws_libuv_kill(context); } static void diff --git a/lib/libwebsockets.h b/lib/libwebsockets.h index 119eee7b..6e8ee448 100644 --- a/lib/libwebsockets.h +++ b/lib/libwebsockets.h @@ -4506,6 +4506,29 @@ lws_uv_sigint_cb(uv_signal_t *watcher, int signum); LWS_VISIBLE LWS_EXTERN void lws_close_all_handles_in_loop(uv_loop_t *loop); + +/* + * Any direct libuv allocations in protocol handlers must participate in the + * lws reference counting scheme. Two apis are provided: + * + * - lws_libuv_static_refcount_add(handle, context) to mark the handle with + * a pointer to the context and increment the global uv object counter + * + * - lws_libuv_static_refcount_del() which should be used as the close callback + * for your own libuv objects declared in the protocol scope. + * + * See the dumb increment plugin for an example of how to use them. + * + * Using the apis allows lws to detach itself from a libuv loop completely + * cleanly and at the moment all of its libuv objects have completed close. + */ + +LWS_VISIBLE LWS_EXTERN void +lws_libuv_static_refcount_add(uv_handle_t *, struct lws_context *context); + +LWS_VISIBLE LWS_EXTERN void +lws_libuv_static_refcount_del(uv_handle_t *); + #endif /* LWS_WITH_LIBUV */ ///@} diff --git a/lib/private-libwebsockets.h b/lib/private-libwebsockets.h index 196724c8..65582731 100644 --- a/lib/private-libwebsockets.h +++ b/lib/private-libwebsockets.h @@ -854,6 +854,7 @@ struct lws_context_per_thread { #if defined(LWS_WITH_LIBEV) || defined(LWS_WITH_LIBUV) || defined(LWS_WITH_LIBEVENT) struct lws_signal_watcher w_sigint; unsigned char ev_loop_foreign:1; + unsigned char event_loop_destroy_processing_done:1; #endif unsigned long count_conns; @@ -1058,6 +1059,25 @@ struct lws_peer { }; #endif +#if defined(LWS_WITH_LIBUV) +/* + * All "static" (per-pt or per-context) uv handles must + * + * - have their .data set to point to the context + * + * - contribute to context->uv_count_static_asset_handles + * counting + */ +#define LWS_UV_REFCOUNT_STATIC_HANDLE_NEW(_x, _ctx) \ + { uv_handle_t *_uht = (uv_handle_t *)(_x); _uht->data = _ctx; \ + _ctx->uv_count_static_asset_handles++; } +#define LWS_UV_REFCOUNT_STATIC_HANDLE_TO_CONTEXT(_x) \ + ((struct lws_context *)((uv_handle_t *)((_x)->data))) +#define LWS_UV_REFCOUNT_STATIC_HANDLE_DESTROYED(_x) \ + (--(LWS_UV_REFCOUNT_STATIC_HANDLE_TO_CONTEXT(_x)-> \ + uv_count_static_asset_handles)) +#endif + /* * the rest is managed per-context, that includes * @@ -1121,6 +1141,7 @@ struct lws_context { #if defined(LWS_WITH_LIBUV) uv_signal_cb lws_uv_sigint_cb; uv_loop_t pu_loop; + int uv_count_static_asset_handles; #endif #if defined(LWS_WITH_LIBEVENT) #if defined(LWS_HIDE_LIBEVENT) @@ -1226,6 +1247,8 @@ lws_adopt_socket_vhost(struct lws_vhost *vh, lws_sockfd_type accept_fd); int lws_jws_base64_enc(const char *in, size_t in_len, char *out, size_t out_max); +void +lws_vhost_destroy1(struct lws_vhost *vh); enum { LWS_EV_READ = (1 << 0), diff --git a/plugins/protocol_dumb_increment.c b/plugins/protocol_dumb_increment.c index e935bf4a..8af2edd6 100644 --- a/plugins/protocol_dumb_increment.c +++ b/plugins/protocol_dumb_increment.c @@ -76,6 +76,8 @@ callback_dumb_increment(struct lws *wsi, enum lws_callback_reasons reason, uv_timer_init(lws_uv_getloop(vhd->context, 0), &vhd->timeout_watcher); + lws_libuv_static_refcount_add((uv_handle_t *)&vhd->timeout_watcher, + vhd->context); uv_timer_start(&vhd->timeout_watcher, uv_timeout_cb_dumb_increment, DUMB_PERIOD, DUMB_PERIOD); @@ -86,7 +88,8 @@ callback_dumb_increment(struct lws *wsi, enum lws_callback_reasons reason, break; lwsl_notice("di: LWS_CALLBACK_PROTOCOL_DESTROY: v=%p, ctx=%p\n", vhd, vhd->context); uv_timer_stop(&vhd->timeout_watcher); - uv_close((uv_handle_t *)&vhd->timeout_watcher, NULL); + uv_close((uv_handle_t *)&vhd->timeout_watcher, + lws_libuv_static_refcount_del); break; case LWS_CALLBACK_ESTABLISHED: diff --git a/test-apps/test-server-libuv.c b/test-apps/test-server-libuv.c index 5aecfa64..c4612e52 100644 --- a/test-apps/test-server-libuv.c +++ b/test-apps/test-server-libuv.c @@ -190,16 +190,6 @@ void outer_signal_cb(uv_signal_t *s, int signum) uv_stop(s->loop); } -static void lws_uv_close_cb(uv_handle_t *handle) -{ - //lwsl_err("%s\n", __func__); -} - -static void lws_uv_walk_cb(uv_handle_t *handle, void *arg) -{ - uv_close(handle, lws_uv_close_cb); -} - /* --- end of foreign test code ---- */ #endif @@ -426,15 +416,14 @@ int main(int argc, char **argv) /* we are here either because signal stopped us, * or outer timer expired */ - /* close short timer */ + /* stop short timer */ uv_timer_stop(&timer_inner); - uv_close((uv_handle_t*)&timer_inner, timer_close_cb); - lwsl_notice("Destroying lws context\n"); /* detach lws */ lws_context_destroy(context); + lws_context_destroy2(context); lwsl_notice("Please wait while the outer libuv test continues for 10s\n"); @@ -452,29 +441,19 @@ int main(int argc, char **argv) uv_timer_stop(&timer_outer); uv_timer_stop(&timer_test_cancel); uv_close((uv_handle_t*)&timer_outer, timer_close_cb); + uv_close((uv_handle_t*)&timer_inner, timer_close_cb); uv_signal_stop(&signal_outer); e = 100; while (e--) uv_run(&loop, UV_RUN_NOWAIT); - /* PHASE 2: close anything remaining */ - - uv_walk(&loop, lws_uv_walk_cb, NULL); - - e = 100; - while (e--) - uv_run(&loop, UV_RUN_NOWAIT); - - /* PHASE 3: close the UV loop itself */ + /* PHASE 2: close the UV loop itself */ e = uv_loop_close(&loop); lwsl_notice("uv loop close rc %s\n", e ? uv_strerror(e) : "ok"); - /* PHASE 4: finalize context destruction */ - - lws_context_destroy2(context); } else #endif { diff --git a/test-apps/test-server-v2.0.c b/test-apps/test-server-v2.0.c index ae81b474..1531beaa 100644 --- a/test-apps/test-server-v2.0.c +++ b/test-apps/test-server-v2.0.c @@ -324,6 +324,7 @@ int main(int argc, char **argv) char ca_path[1024] = ""; int uid = -1, gid = -1; int use_ssl = 0; + uv_loop_t loop; int opts = 0; int n = 0; #ifndef _WIN32 @@ -433,14 +434,18 @@ int main(int argc, char **argv) openlog("lwsts", syslog_options, LOG_DAEMON); #endif - /* tell the library what debug level to emit and to send it to syslog */ - lws_set_log_level(debug_level, lwsl_emit_syslog); + /* tell the library what debug level to emit */ + lws_set_log_level(debug_level, NULL); lwsl_notice("libwebsockets test server - license LGPL2.1+SLE\n"); lwsl_notice("(C) Copyright 2010-2017 Andy Green \n"); lwsl_notice(" Using resource path \"%s\"\n", resource_path); + uv_loop_init(&loop); +#if defined(TEST_DYNAMIC_VHOST) + uv_timer_init(&loop, &timeout_watcher); +#endif info.iface = iface; info.protocols = NULL; /* all protocols from lib / plugins */ info.ssl_cert_filepath = NULL; @@ -534,25 +539,31 @@ int main(int argc, char **argv) /* libuv event loop */ lws_uv_sigint_cfg(context, 1, signal_cb); - if (lws_uv_initloop(context, NULL, 0)) { + if (lws_uv_initloop(context, &loop, 0)) { lwsl_err("lws_uv_initloop failed\n"); goto bail; } -#if defined(TEST_DYNAMIC_VHOST) - uv_timer_init(lws_uv_getloop(context, 0), &timeout_watcher); -#endif lws_libuv_run(context, 0); -#if defined(TEST_DYNAMIC_VHOST) - uv_timer_stop(&timeout_watcher); - uv_close((uv_handle_t *)&timeout_watcher, NULL); -#endif - bail: /* when we decided to exit the event loop */ lws_context_destroy(context); lws_context_destroy2(context); + + +#if defined(TEST_DYNAMIC_VHOST) + uv_timer_stop(&timeout_watcher); + uv_close((uv_handle_t *)&timeout_watcher, NULL); + + /* let it run until everything completed close */ + uv_run(&loop, UV_RUN_DEFAULT); +#endif + + /* nothing left in the foreign loop, destroy it */ + + uv_loop_close(&loop); + lwsl_notice("libwebsockets-test-server exited cleanly\n"); #ifndef _WIN32