examples: move existing to m-e-lowlevel and start repoulating m-e with SS

CMakeLists.txt

@@ -942,7 +942,7 @@ endif()
 
 if (LWS_HAVE_LIBCAP)
 	find_library(LIBCAP_LIBRARIES NAMES cap)
-	list(APPEND LIB_LIST ${LIBCAP_LIBRARIES} )
+	list(APPEND LIB_LIST_AT_END ${LIBCAP_LIBRARIES} )
 endif()
 
 if (LWS_WITH_PLUGINS_BUILTIN)
@@ -1006,7 +1006,10 @@ set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
 message("DIR ${libwebsockets_DIR} CMP ${CMAKE_MODULE_PATH}")
 
 if (LWS_WITH_MINIMAL_EXAMPLES)
-	add_subdirectory(minimal-examples)
+	if (LWS_WITH_SECURE_STREAMS)
+		add_subdirectory(minimal-examples)
+	endif()
+	add_subdirectory(minimal-examples-lowlevel)
 endif()
 
 if (NOT LWS_WITHOUT_TESTAPPS)

README.md

@@ -26,6 +26,84 @@ sdevent, glib and uloop, as well as custom event libs.
 News
 ----
 
+## Lws examples switching to Secure Streams
+
+![Secure Streams direct](./doc-assets/ss-api1.png)
+
+**Secure Streams** support in lws was introduced a couple of years ago, it's a
+higher-level interface to lws `wsi`-level apis that simplifies connectivity by
+segregating connection policy like protocol and endpoint information into a
+separate [JSON policy file](./minimal-examples/client/hello_world/example-policy.json), and just having the [code deal with payloads](./minimal-examples/clients/hello_world/hello_world-ss.c); as many
+details of the wire protocol as possible are hidden or moved to the policy, so
+user code is almost identical even if the wire protocol changes.
+
+The user code just asks to create a SS by "streamtype name", it is created
+according to the details (protocol, endpoint, etc) under the same name in the
+policy.
+
+Key policy entries like endpoint can contain `${metadata-name}` string
+substitutions to handle runtime adaptations via metadata.  h1, h2, ws and mqtt
+are supported.
+
+As a layer on top of the `wsi` apis, SS provides a higher-level way to access
+the existing wsi-level capabilities, both kinds of API will remain supported.
+Secure Streams are longer-lived than a single wsi, so an SS can coordinate
+retries by itself.  SS-based user code is typically significantly smaller and
+more maintainable than wsi layer.
+
+In main branch I have moved the older examples into `./minimal-examples-lowlevel`
+and am starting to port more cases from there into SS-based examples.
+
+### Comparison between wsi and SS level lws usage
+
+|Feature|"low-level" wsi way|Secure Streams way|
+|---|---|---|
+|Create context|code|same|
+|Loop support, sul scheduler|default, event libs|same|
+|Supports comms mode|Client, Server, Raw|same|
+|Supports protocols|h1, h2, ws, mqtt (client)|same|
+|TLS support|mbedtls (including v3), openssl (including v3), wolfssl, boringssl, libressl|same|
+|Serializable, proxiable, muxable, transportable|No|Yes|
+|Auto-allocated per-connection user object|pss specified in lws_protocols|Specified in ss info struct|
+|Connection User API|Protocol-specific lws_protocols cbs (> 100)|SS API (rx, tx, state callbacks only)|
+|Sending adaptation|lws_callback_on_writeable()  + WRITEABLE|lws_ss_request_write() + tx() cb|
+|Sending buffer|User-chosen + malloc'd partial handling|SS-provided, no partials|
+|Create vhosts|code|**JSON policy**|
+|TLS validation|cert bundle or code|**JSON policy**, or cert bundle|
+|Connection retry / backoff|code|**JSON policy**, Auto|
+|Nailing up|code|**JSON policy**, Auto|
+|Endpoint and protocol details|spread around the code|**JSON policy**|
+|Protocol selection, pipeline / stream sharing|code|**JSON policy**|
+|ws subprotocol selection|code|**JSON policy**|
+|ws binary / text|code|**JSON policy**|
+|Protocol-specific metadata|Protocol-specific apis in code (eg, lws_hdr)|**JSON policy**, generic metadata apis in code|
+|Connection validity rules|struct|**JSON policy**, Auto|
+|Stream as Long Poll|code|**JSON policy**|
+|Auth|code|**JSON policy** + automatic rotation if provider supported, else code|
+
+### Serialized Secure Streams
+
+![Secure Streams direct](./doc-assets/ss-api2.png)
+
+Secure Streams APIs are also **serializable**, the exact same client code can
+fulfil the connection directly in the same process as you would expect, or
+forward the actions, metadata and payloads to an [SS Proxy](./minimal-examples/ssproxy/ssproxy-socket) that owns the policy
+over a Unix Domain or TCP socket connection to be fulfilled centrally.  This
+allows, eg, h2 streams from different processes sharing a single connection.
+
+![Secure Streams direct](./doc-assets/ss-api3.png)
+
+The serialized SS can also travel over generic transports like UART, an [example
+is provided implementing the Binance example on an RPi Pico](./minimal-examples/embedded/pico/pico-sspc-binance) with a UART transport
+to a [UART transport SS proxy](./minimal-examples/ssproxy/ssproxy-custom-transport-uart), where the pico itself has no network stack, tls, compression or
+wss stack, but can send and receive to and from the endpoint as if it did.
+
+The optional `lws_trasport_mux` is used to interpose between the UART transport
+and the SSPC layer, allowing a single pipe to carry many separate SS connections.
+
+The user SS code is identical however it is transported, muxed and fulfilled.
+
+
 ## v4.3 is released
 
 See the [changelog](https://libwebsockets.org/git/libwebsockets/tree/changelog)

READMEs/README.ctest.md

@@ -282,7 +282,7 @@ named fixture "hcm_srv", itself with an 800s timeout
 
 ```
 	set_tests_properties(st_hcm_srv PROPERTIES
-       		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/minimal-examples/http-server/minimal-http-server-tls
+       		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/minimal-examples-lowlevel/http-server/minimal-http-server-tls
 		FIXTURES_SETUP hcm_srv
 		TIMEOUT 800)
 	set_tests_properties(ki_hcm_srv PROPERTIES
@@ -296,7 +296,7 @@ test (http-client-multi) we are testing
 	set_tests_properties(http-client-multi
 			     PROPERTIES
 			     FIXTURES_REQUIRED "hcm_srv"
-			     WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/minimal-examples/http-client/minimal-http-client-multi
+			     WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/minimal-examples-lowlevel/http-client/minimal-http-client-multi
 			     TIMEOUT 50)
 ```
 

include/libwebsockets/lws-context-vhost.h

@@ -1229,6 +1229,19 @@ lws_context_user(struct lws_context *context);
 LWS_VISIBLE LWS_EXTERN const char *
 lws_vh_tag(struct lws_vhost *vh);
 
+LWS_VISIBLE LWS_EXTERN void
+_lws_context_info_defaults(struct lws_context_creation_info *info,
+			   const char *sspol);
+
+LWS_VISIBLE LWS_EXTERN void
+lws_default_loop_exit(struct lws_context *cx);
+
+LWS_VISIBLE LWS_EXTERN void
+lws_context_default_loop_run_destroy(struct lws_context *cx);
+
+LWS_VISIBLE LWS_EXTERN int
+lws_cmdline_passfail(int argc, const char **argv, int actual);
+
 /**
  * lws_context_is_being_destroyed() - find out if context is being destroyed
  *

include/libwebsockets/lws-misc.h

@@ -886,7 +886,7 @@ LWS_VISIBLE extern const lws_humanize_unit_t humanize_schema_us[8];
 void
 lws_assert_fourcc(uint32_t fourcc, uint32_t expected);
 #else
-#define lws_assert_fourcc(_a, _b)
+#define lws_assert_fourcc(_a, _b) do { } while (0);
 #endif
 
 /**

include/libwebsockets/lws-secure-streams-client.h

@@ -66,6 +66,7 @@ struct lws_sspc_handle;
 #define lws_ss_to_user_object		lws_sspc_to_user_object
 #define lws_ss_change_handlers		lws_sspc_change_handlers
 #define lws_smd_ss_rx_forward		lws_smd_sspc_rx_forward
+#define lws_ss_server_ack		lws_sspc_server_ack
 #define lws_ss_tag			lws_sspc_tag
 #define _lws_fi_user_ss_fi		_lws_fi_user_sspc_fi
 #define lwsl_ss_get_cx			lwsl_sspc_get_cx
@@ -349,6 +350,9 @@ LWS_VISIBLE LWS_EXTERN void
 lws_sspc_change_handlers(struct lws_sspc_handle *h,
 			 lws_sscb_rx rx,lws_sscb_tx tx, lws_sscb_state state);
 
+LWS_VISIBLE LWS_EXTERN void
+lws_sspc_server_ack(struct lws_sspc_handle *h, int nack);
+
 
 /*
  * Helpers offered by lws to handle transport SSPC-side proxy link events

include/libwebsockets/lws-secure-streams.h

@@ -248,6 +248,28 @@ typedef struct lws_ss_info {
 
 } lws_ss_info_t;
 
+#define LWS_SS_USER_TYPEDEF \
+	typedef struct { \
+		struct lws_ss_handle 	*ss; \
+		void			*opaque_data;
+
+#define LWS_SS_INFO(_streamtype, _type) \
+	const lws_ss_info_t ssi_##_type = { \
+		.handle_offset = offsetof(_type, ss), \
+		.opaque_user_data_offset = offsetof(_type, opaque_data), \
+		.user_alloc = sizeof(_type), \
+		.streamtype = _streamtype,
+
+#define lws_ss_from_user(_u)		(_u)->ss
+#define lws_ss_opaque_from_user(_u)	(_u)->opaque_data
+#define lws_ss_cx_from_user(_u)		lws_ss_get_context((_u)->ss)
+
+#if defined(LWS_SS_USE_SSPC)
+#define lws_context_info_defaults(_x, _y) _lws_context_info_defaults(_x, NULL)
+#else
+#define lws_context_info_defaults(_x, _y) _lws_context_info_defaults(_x, _y)
+#endif
+
 /**
  * lws_ss_create() - Create secure stream
  *

lib/core-net/vhost.c

@@ -690,10 +690,10 @@ lws_create_vhost(struct lws_context *context,
 	if (!pcols) {
 		for (vh->count_protocols = 0;
 			info->pprotocols[vh->count_protocols];
-			vh->count_protocols++) {
-				lwsl_user("%s: ppcols: %s\n", __func__,
-						info->pprotocols[vh->count_protocols]->name);
-		}
+			vh->count_protocols++)
+				;
+			//lwsl_user("%s: ppcols: %s\n", __func__,
+			// info->pprotocols[vh->count_protocols]->name);
 	} else
 		for (vh->count_protocols = 0;
 			pcols[vh->count_protocols].callback;

lib/core/libwebsockets.c

@@ -1399,7 +1399,10 @@ static const char * const builtins[] = {
 	"-d",
 	"--fault-injection",
 	"--fault-seed",
-	"--ignore-sigterm"
+	"--ignore-sigterm",
+	"--ssproxy-port",
+	"--ssproxy-iface",
+	"--ssproxy-ads",
 };
 
 enum opts {
@@ -1407,6 +1410,9 @@ enum opts {
 	OPT_FAULTINJECTION,
 	OPT_FAULT_SEED,
 	OPT_IGNORE_SIGTERM,
+	OPT_SSPROXY_PORT,
+	OPT_SSPROXY_IFACE,
+	OPT_SSPROXY_ADS,
 };
 
 #if !defined(LWS_PLAT_FREERTOS)
@@ -1416,6 +1422,72 @@ lws_sigterm_catch(int sig)
 }
 #endif
 
+void
+_lws_context_info_defaults(struct lws_context_creation_info *info,
+			  const char *sspol)
+{
+	memset(info, 0, sizeof *info);
+        info->fd_limit_per_thread = 1 + 6 + 1;
+#if defined(LWS_WITH_NETWORK)
+        info->port = CONTEXT_PORT_NO_LISTEN;
+#endif
+#if defined(LWS_WITH_SECURE_STREAMS) && !defined(LWS_WITH_SECURE_STREAMS_STATIC_POLICY_ONLY)
+        info->pss_policies_json = sspol;
+#endif
+#if defined(LWS_WITH_SECURE_STREAMS_PROXY_API)
+        if (!sspol)
+        	info->protocols = lws_sspc_protocols;
+        else
+#endif
+        	info->options = LWS_SERVER_OPTION_EXPLICIT_VHOSTS |
+        		LWS_SERVER_OPTION_H2_JUST_FIX_WINDOW_UPDATE_OVERFLOW |
+        		LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT;
+}
+
+void
+lws_default_loop_exit(struct lws_context *cx)
+{
+	if (cx) {
+		cx->interrupted = 1;
+#if defined(LWS_WITH_NETWORK)
+		lws_cancel_service(cx);
+#endif
+	}
+}
+
+#if defined(LWS_WITH_NETWORK)
+void
+lws_context_default_loop_run_destroy(struct lws_context *cx)
+{
+        /* the default event loop, since we didn't provide an alternative one */
+
+        while (!cx->interrupted && lws_service(cx, 0) >= 0)
+        	;
+
+        lws_context_destroy(cx);
+}
+#endif
+
+int
+lws_cmdline_passfail(int argc, const char **argv, int actual)
+{
+	int expected = 0;
+	const char *p;
+
+	if ((p = lws_cmdline_option(argc, argv, "--expected-exit")))
+		expected = atoi(p);
+
+	if (actual == expected) {
+		lwsl_user("Completed: OK (seen expected %d)\n", actual);
+
+		return 0;
+	}
+
+	lwsl_err("Completed: failed: exit %d, expected %d\n", actual, expected);
+
+	return 1;
+}
+
 void
 lws_cmdline_option_handle_builtin(int argc, const char **argv,
 				  struct lws_context_creation_info *info)
@@ -1438,6 +1510,25 @@ lws_cmdline_option_handle_builtin(int argc, const char **argv,
 			logs = m;
 			break;
 
+#if defined(LWS_WITH_SECURE_STREAMS_PROXY_API)
+		case OPT_SSPROXY_PORT:
+			/* connect to ssproxy via UDS by default, else via
+			 * tcp connection to this port */
+			info->ss_proxy_port = (uint16_t)atoi(p);
+			break;
+
+		case OPT_SSPROXY_IFACE:
+			/* UDS "proxy.ss.lws" in abstract namespace, else this socket
+			 * path; when -p given this can specify the network interface
+			 * to bind to */
+			info->ss_proxy_bind = p;
+			break;
+
+		case OPT_SSPROXY_ADS:
+			info->ss_proxy_address = p;
+			break;
+#endif
+
 		case OPT_FAULTINJECTION:
 #if !defined(LWS_WITH_SYS_FAULT_INJECTION)
 			lwsl_err("%s: FAULT_INJECTION not built\n", __func__);

lib/core/private-lib-core.h

@@ -735,6 +735,7 @@ struct lws_context {
 	char		tls_gate_accepts;
 
 	unsigned int deprecated:1;
+	unsigned int interrupted:1;
 	unsigned int inside_context_destroy:1;
 	unsigned int being_destroyed:1;
 	unsigned int service_no_longer_possible:1;

lib/secure-streams/policy-json.c

@@ -395,6 +395,7 @@ lws_ss_policy_parser_cb(struct lejp_ctx *ctx, char reason)
 		 * The struct *x is in the lwsac... the ca_der it points to
 		 * is individually allocated from the heap
 		 */
+
 		a->curr[LTY_X509].x->ca_der = lws_malloc((unsigned int)a->count, "ssx509");
 		if (!a->curr[LTY_X509].x->ca_der)
 			goto oom;
@@ -1168,7 +1169,7 @@ lws_ss_policy_parse_abandon(struct lws_context *context)
 {
 	struct policy_cb_args *args = (struct policy_cb_args *)context->pol_args;
 	lws_ss_x509_t *x;
-
+lwsl_notice("%s\n", __func__);
 	x = args->heads[LTY_X509].x;
 	while (x) {
 		/*
@@ -1206,8 +1207,10 @@ lws_ss_policy_parse_file(struct lws_context *cx, const char *filepath)
 	uint8_t buf[512];
 	int n, m, fd = lws_open(filepath, LWS_O_RDONLY);
 
-	if (fd < 0)
+	if (fd < 0) {
+		lwsl_cx_err(cx, "Unable to open policy '%s'", filepath);
 		return LEJP_REJECT_UNKNOWN;
+	}
 
 	do {
 		n = (int)read(fd, buf, sizeof(buf));

lib/secure-streams/protocols/ss-h1.c

@@ -549,7 +549,7 @@ secstream_h1(struct lws *wsi, enum lws_callback_reasons reason, void *user,
 		if (h->ss_dangling_connected) {
 			/* already disconnected, no action for DISCONNECT_ME */
 			r = lws_ss_event_helper(h, LWSSSCS_DISCONNECTED);
-			if (r != LWSSSSRET_OK)
+			if (r == LWSSSSRET_DESTROY_ME)
 				return _lws_ss_handle_state_ret_CAN_DESTROY_HANDLE(r, wsi, &h);
 		}
 		break;
@@ -919,7 +919,7 @@ malformed:
 	case LWS_CALLBACK_CLIENT_HTTP_WRITEABLE:
 
 		if (!h || !h->info.tx) {
-			lwsl_notice("%s: no handle / tx\n", __func__);
+			lwsl_debug("%s: no handle / tx\n", __func__);
 			return 0;
 		}
 

lib/secure-streams/secure-streams.c

@@ -433,6 +433,7 @@ lws_ss_event_helper(lws_ss_handle_t *h, lws_ss_constate_t cs)
 			cs == LWSSSCS_UNREACHABLE &&
 			h->wsi && h->wsi->dns_reachability);
 		h->h_in_svc = NULL;
+
 #if defined(LWS_WITH_SERVER)
 		if ((h->info.flags & LWSSSINFLAGS_ACCEPTED) &&
 		    cs == LWSSSCS_DISCONNECTED)
@@ -1502,7 +1503,7 @@ lws_ss_destroy(lws_ss_handle_t **ppss)
 #endif
 
 #if defined(LWS_WITH_SERVER)
-	if (v)
+	if (v && (h->info.flags & LWSSSINFLAGS_SERVER))
 		/*
 		 * For server, the policy describes a vhost that implements the
 		 * server, when we take down the ss, we take down the related

lib/secure-streams/serialized/client/sspc-transport.c

@@ -173,9 +173,6 @@ lws_sspc_txp_event_closed(lws_transport_priv_t priv)
 	lws_sspc_handle_t *h = (lws_sspc_handle_t *)priv;
 	lws_ss_state_return_t r = LWSSSSRET_OK;
 
-
-	lwsl_sspc_notice(h, "entry");
-
 	if (!h) {
 		lwsl_sspc_info(h, "No sspc on client proxy link close");
 		return LWSSSSRET_OK;
@@ -275,7 +272,7 @@ lws_sspc_txp_tx(lws_sspc_handle_t *h, size_t metadata_limit)
 		 * We are negotating the opening of a particular
 		 * streamtype
 		 */
-		lwsl_sspc_notice(h, "LPCSCLI_SENDING_INITIAL_TX");
+		// lwsl_sspc_notice(h, "LPCSCLI_SENDING_INITIAL_TX");
 		txl = strlen(h->ssi.streamtype) + 1 + 4 + 4;
 
 		cp = s;
@@ -297,7 +294,7 @@ lws_sspc_txp_tx(lws_sspc_handle_t *h, size_t metadata_limit)
 
 	case LPCSCLI_LOCAL_CONNECTED:
 
-		lwsl_sspc_notice(h, "LPCSCLI_LOCAL_CONNECTED");
+		// lwsl_sspc_notice(h, "LPCSCLI_LOCAL_CONNECTED");
 
 		/*
 		 * Do we need to prioritize sending any metadata

lib/secure-streams/serialized/client/sspc.c

@@ -339,8 +339,8 @@ lws_sspc_create(struct lws_context *context, int tsi, const lws_ss_info_t *ssi,
 
 	/* priv_onw filled in by onw transport */
 
-	lwsl_sspc_notice(h, "txp path %s -> %s", h->txp_path.ops_in->name,
-						 h->txp_path.ops_onw->name);
+	lwsl_sspc_info(h, "txp path %s -> %s", h->txp_path.ops_in->name,
+					       h->txp_path.ops_onw->name);
 
 	memcpy(&h->ssi, ssi, sizeof(*ssi));
 	ua = (uint8_t *)&h[1];
@@ -502,7 +502,7 @@ lws_sspc_request_tx(lws_sspc_handle_t *h)
 	if (!h->us_earliest_write_req)
 		h->us_earliest_write_req = lws_now_usecs();
 
-	lwsl_notice("%s: state %u, conn_req_state %u\n", __func__,
+	lwsl_info("%s: state %u, conn_req_state %u\n", __func__,
 			(unsigned int)h->state,
 			(unsigned int)h->conn_req_state);
 
@@ -697,6 +697,14 @@ _lws_sspc_set_metadata(struct lws_sspc_handle *h, const char *name,
 	return 0;
 }
 
+void
+lws_sspc_server_ack(struct lws_sspc_handle *h, int nack)
+{
+	//h->txn_resp = nack;
+	//h->txn_resp_set = 1;
+
+}
+
 int
 lws_sspc_set_metadata(struct lws_sspc_handle *h, const char *name,
 		      const void *value, size_t len)

lib/tls/tls-network.c

@@ -201,11 +201,11 @@ lws_gate_accepts(struct lws_context *context, int on)
 {
 	struct lws_vhost *v = context->vhost_list;
 
-	lwsl_notice("%s: on = %d\n", __func__, on);
-
 	if (context->tls_gate_accepts == (char)on)
 		return 0;
 
+	lwsl_notice("%s: on = %d\n", __func__, on);
+
 	context->tls_gate_accepts = (char)on;
 
 	while (v) {

minimal-examples-lowlevel/CMakeLists.txt

@@ -0,0 +1,50 @@
+#
+# libwebsockets - small server side websockets and web server implementation
+#
+# Copyright (C) 2010 - 2021 Andy Green <andy@warmcat.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+MACRO(SUBDIRLIST result curdir)
+  FILE(GLOB children RELATIVE ${curdir} ${curdir}/*)
+  SET(dirlist "")
+
+  FOREACH(child ${children})
+    IF (IS_DIRECTORY ${curdir}/${child})
+	LIST(APPEND dirlist ${child})
+    ENDIF()
+  ENDFOREACH()
+
+  SET(${result} ${dirlist})
+ENDMACRO()
+
+include_directories(${LWS_LIB_BUILD_INC_PATHS})
+link_libraries(${LIB_LIST_AT_END})
+
+SUBDIRLIST(SUBDIRS "${PROJECT_SOURCE_DIR}/minimal-examples-lowlevel")
+FOREACH(subdir ${SUBDIRS})
+
+	SUBDIRLIST(SUBDIRS2 "${PROJECT_SOURCE_DIR}/minimal-examples-lowlevel/${subdir}")
+	FOREACH(subdir2 ${SUBDIRS2})
+		if (EXISTS "${PROJECT_SOURCE_DIR}/minimal-examples-lowlevel/${subdir}/${subdir2}/CMakeLists.txt")
+			message("Processing ${PROJECT_SOURCE_DIR}/minimal-examples-lowlevel/${subdir}/${subdir2}")
+			add_subdirectory("${PROJECT_SOURCE_DIR}/minimal-examples-lowlevel/${subdir}/${subdir2}")
+		endif()
+	ENDFOREACH()
+ENDFOREACH()

minimal-examples-lowlevel/README.md

@@ -0,0 +1,100 @@
+## minimal-examples-lowlevel
+
+These are the traditional lws low-level, wsi-based examples.
+
+`./minimal-examples` contains higher-level, Secure Stream based examples for a
+growing subset of these cases.  New users may find it easier to use those higher-
+level examples for the (common) cases they cover.
+
+These lowlevel apis are not going anywhere, the higher level stuff uses them
+to get stuff done itself. 
+
+|name|demonstrates|
+---|---
+client-server|Minimal examples providing client and server connections simultaneously
+crypto|Minimal examples related to using lws crypto apis
+dbus-server|Minimal examples showing how to integrate DBUS into lws event loop
+http-client|Minimal examples providing an http client
+http-server|Minimal examples providing an http server
+raw|Minimal examples related to adopting raw file or socket descriptors into the event loop
+secure-streams|Minimal examples related to the Secure Streams client api
+ws-client|Minimal examples providing a ws client
+ws-server|Minimal examples providing a ws server (and an http server)
+
+## FAQ
+
+### Getting started
+
+Build and install lws itself first (note that after installing lws on \*nix, you need to run `ldconfig` one time so the OS can learn about the new library.  Lws installs in `/usr/local` by default, Debian / Ubuntu ldconfig knows to look there already, but Fedora / CentOS need you to add the line `/usr/local/lib` to `/etc/ld.so.conf` and run ldconfig)
+
+Then start with the simplest:
+
+`http-server/minimal-http-server`
+
+### Why are most of the sources split into a main C file file and a protocol file?
+
+Lws supports three ways to implement the protocol callback code:
+
+ - you can just add it all in the same source file
+
+ - you can separate it as these examples do, and #include it
+   into the main sources
+
+ - you can build it as a standalone plugin that is discovered
+   and loaded at runtime.
+
+The way these examples are structured, you can easily also build
+the protocol callback as a plugin just with a different
+CMakeLists.txt... see https://github.com/warmcat/libwebsockets/tree/master/plugin-standalone
+for an example.
+
+### Why would we want the protocol as a plugin?
+
+You will notice a lot of the main C code is the same boilerplate
+repeated for each example.  The actual interesting part is in
+the protocol callback only.
+
+Lws provides (-DLWS_WITH_LWSWS=1) a generic lightweight server app called 'lwsws' that
+can be configured by JSON.  Combined with your protocol as a plugin,
+it means you don't actually have to make a special server "app"
+part, you can just use lwsws and pass per-vhost configuration
+from JSON into your protocol.  (Of course in some cases you have
+an existing app you are bolting lws on to, then you don't care
+about this for that particular case).
+
+Because lwsws has no dependency on whatever your plugin does, it
+can mix and match different protocols randomly without needing any code
+changes.  It reduces the size of the task to just writing the
+code you care about in your protocol handler, and nothing else to write
+or maintain.
+
+Lwsws supports advanced features like reload, where it starts a new server
+instance with changed config or different plugins, while keeping the old
+instance around until the last connection to it closes.
+
+### I get why there is a pss, but why is there a vhd?
+
+The pss is instantiated per-connection.  But there are almost always
+other variables that have a lifetime longer than a single connection.
+
+You could make these variables "filescope" one-time globals, but that
+means your protocol cannot instantiate multiple times.
+
+Lws supports vhosts (virtual hosts), for example both https://warmcat.com
+and https://libwebsockets are running on the same lwsws instance on the
+same server and same IP... each of these is a separate vhost.
+
+Your protocol may be enabled on multiple vhosts, each of these vhosts
+provides a different vhd specific to the protocol instance on that
+vhost.  For example many of the samples keep a linked-list head to
+a list of live pss in the vhd... that means it's cleanly a list of
+pss opened **on that vhost**.  If another vhost has the protocol
+enabled, connections to that will point to a different vhd, and the
+linked-list head on that vhd will only list connections to his vhost.
+
+The example "ws-server/minimal-ws-server-threads" demonstrates how to deliver
+external configuration data to a specific vhost + protocol
+combination using code.  In lwsws, this is simply a matter of setting
+the desired JSON config.
+
+

minimal-examples-lowlevel/api-tests/api-test-async-dns/CMakeLists.txt

@@ -18,7 +18,7 @@ if (requirements)
 	add_test(NAME api-test-async-dns COMMAND lws-api-test-async-dns)
 	set_tests_properties(api-test-async-dns
 			     PROPERTIES
-			     WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/minimal-examples/api-tests/api-test-async-dns
+			     WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/minimal-examples-lowlevel/api-tests/api-test-async-dns
 			     TIMEOUT 60)
 
 	if (websockets_shared)

minimal-examples-lowlevel/api-tests/api-test-lws_smd/CMakeLists.txt

@@ -16,7 +16,7 @@ if (requirements)
 	set_tests_properties(api-test-lws_smd
 			     PROPERTIES
 			     RUN_SERIAL TRUE
-			     WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/minimal-examples/api-tests/api-test-lws_smd
+			     WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/minimal-examples-lowlevel/api-tests/api-test-lws_smd
 			     TIMEOUT 60)
 
 	if (websockets_shared)

minimal-examples-lowlevel/api-tests/api-test-secure-streams/CMakeLists.txt

@@ -17,7 +17,7 @@ if (requirements)
 		add_test(NAME api-test-secure-streams COMMAND ${PROJECT_NAME})
 		set_tests_properties(api-test-secure-streams
 				     PROPERTIES
-				     WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/minimal-examples/api-tests/api-test-secure-streams
+				     WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/minimal-examples-lowlevel/api-tests/api-test-secure-streams
 				     TIMEOUT 20)
 	endif()