diff --git a/lib/core/context.c b/lib/core/context.c
index f52d7e7d8..b7e379fd3 100644
--- a/lib/core/context.c
+++ b/lib/core/context.c
@@ -66,19 +66,6 @@ lws_sul_stats_cb(lws_sorted_usec_list_t *sul)
 			    &pt->sul_stats, 10 * LWS_US_PER_SEC);
 }
 #endif
-#if defined(LWS_WITH_PEER_LIMITS)
-static void
-lws_sul_peer_limits_cb(lws_sorted_usec_list_t *sul)
-{
-	struct lws_context_per_thread *pt = lws_container_of(sul,
-			struct lws_context_per_thread, sul_peer_limits);
-
-	lws_peer_cull_peer_wait_list(pt->context);
-
-	__lws_sul_insert_us(&pt->pt_sul_owner[LWSSULLI_MISS_IF_SUSPENDED],
-			    &pt->sul_peer_limits, 10 * LWS_US_PER_SEC);
-}
-#endif
 
 #if defined(LWS_WITH_NETWORK)
 
@@ -1170,11 +1157,6 @@ lws_create_context(const struct lws_context_creation_info *info)
 	__lws_sul_insert_us(&context->pt[0].pt_sul_owner[LWSSULLI_MISS_IF_SUSPENDED],
 			    &context->pt[0].sul_stats, 10 * LWS_US_PER_SEC);
 #endif
-#if defined(LWS_WITH_PEER_LIMITS)
-	context->pt[0].sul_peer_limits.cb = lws_sul_peer_limits_cb;
-	__lws_sul_insert_us(&context->pt[0].pt_sul_owner[LWSSULLI_MISS_IF_SUSPENDED],
-			    &context->pt[0].sul_peer_limits, 10 * LWS_US_PER_SEC);
-#endif
 
 #if defined(LWS_HAVE_SYS_CAPABILITY_H) && defined(LWS_HAVE_LIBCAP)
 	memcpy(context->caps, info->caps, sizeof(context->caps));
diff --git a/lib/misc/peer-limits.c b/lib/misc/peer-limits.c
index 2f40c2ed0..f7845464b 100644
--- a/lib/misc/peer-limits.c
+++ b/lib/misc/peer-limits.c
@@ -39,11 +39,26 @@ __lws_peer_remove_from_peer_wait_list(struct lws_context *context,
 			*p = df->peer_wait_list;
 			df->peer_wait_list = NULL;
 
+			if (!context->peer_wait_list)
+				lws_sul_cancel(&context->pt[0].sul_peer_limits);
+
 			return;
 		}
 	} lws_end_foreach_llp(p, peer_wait_list);
 }
 
+void
+lws_sul_peer_limits_cb(lws_sorted_usec_list_t *sul)
+{
+	struct lws_context_per_thread *pt = lws_container_of(sul,
+			struct lws_context_per_thread, sul_peer_limits);
+
+	lws_peer_cull_peer_wait_list(pt->context);
+
+	lws_sul_schedule(pt->context, 0, &pt->context->pt[0].sul_peer_limits,
+			 lws_sul_peer_limits_cb, 10 * LWS_US_PER_SEC);
+}
+
 /* requires context->lock */
 static void
 __lws_peer_add_to_peer_wait_list(struct lws_context *context,
@@ -53,6 +68,10 @@ __lws_peer_add_to_peer_wait_list(struct lws_context *context,
 
 	peer->peer_wait_list = context->peer_wait_list;
 	context->peer_wait_list = peer;
+
+	if (!context->pt[0].sul_peer_limits.list.owner)
+		lws_sul_schedule(context, 0, &context->pt[0].sul_peer_limits,
+				lws_sul_peer_limits_cb, 10 * LWS_US_PER_SEC);
 }