remove fixed rx buffer allow definition per protocol

A new protocol member is defined that controls the size of rx
buffer allocation per connection.  For compatibility 0 size
allocates 4096, but you should adapt your protocol definition
array in the user code to declare an appropriate value.

See the changelog for more detail.

The advantage is the rx frame buffer size is now tailored to
what is expected from the protocol, rather than being fixed
to a default of 4096.  If your protocol only sends frames of
a dozen bytes this allows you to only allocate an rx frame
buffer of the same size.

For example the per-connection allocation (excluding headers)
for the test server fell from ~4500 to < 750 bytes with this.

Signed-off-by: Andy Green <andy.green@linaro.org>

README.build

@@ -159,13 +159,6 @@ tradeoff between taking too much and needless realloc
  - LWS_ADDITIONAL_HDR_ALLOC default 64: how much to additionally realloc if
 the header value string keeps coming
 
- - MAX_USER_RX_BUFFER default 4096: max amount of user rx data to buffer at a
-time and pass to user callback LWS_CALLBACK_RECEIVE or
-LWS_CALLBACK_CLIENT_RECEIVE.  Large frames are passed to the user callback
-in chunks of this size.  Tradeoff between per-connection static memory
-allocation and if you expect to deal with large frames, how much you can
-see at once which can affect efficiency.
-
  - LWS_MAX_PROTOCOLS default 10: largest amount of different protocols the
 server can serve
 

changelog

@@ -15,9 +15,35 @@ User api changes
 ----------------
 
  - Header tokens are now deleted after the websocket connection is
- established.  Not just the header data is saved, but the pointer and length
- array is also removed from (union) scope saving several hundred bytes per
- connection once it is established
+	established.  Not just the header data is saved, but the pointer and
+	length array is also removed from (union) scope saving several hundred
+	bytes per connection once it is established
+
+ - struct libwebsocket_protocols has a new member rx_buffer_size, this
+	controls rx buffer size per connection of that protocol now.  Sources
+	for apps built against older versions of the library won't declare
+	this in their protocols, defaulting it to 0.  Zero buffer is legal,
+	it causes a default buffer to be allocated (currently 4096)
+
+	If you want to receive only atomic frames in your user callback, you
+	should set this to greater than your largest frame size.  If a frame
+	comes that exceeds that, no error occurs but the callback happens as
+	soon as the buffer limit is reached, and again if it is reached again
+	or the frame completes.  You can detect that has happened by seeing
+	there is still frame content pending using
+	libwebsockets_remaining_packet_payload()
+
+	By correctly setting this, you can save a lot of memory when your
+	protocol has small frames (see the test server and client sources).
+
+
+User api removals
+-----------------
+
+The configuration-time option MAX_USER_RX_BUFFER has been replaced by a
+buffer size chosen per-protocol.  For compatibility, there's a default of
+4096 rx buffer, but user code should set the appropriate size for the
+protocol frames.
 
 
 New features
@@ -28,6 +54,9 @@ the visual studio project files that were in the tree until now.
 
  - PATH_MAX or MAX_PATH no longer needed
 
+ - cutomizable frame rx buffer size by protocol
+
+
 
 v1.1-chrome26-firefox18
 =======================

lib/client-parser.c

@@ -218,6 +218,10 @@ int libwebsocket_client_rx_sm(struct libwebsocket *wsi, unsigned char c)
 		break;
 
 	case LWS_RXPS_PAYLOAD_UNTIL_LENGTH_EXHAUSTED:
+
+		if (!wsi->u.ws.rx_user_buffer)
+			lwsl_err("NULL client rx_user_buffer\n");
+
 		if ((!wsi->u.ws.this_frame_masked) || wsi->u.ws.all_zero_nonce)
 			wsi->u.ws.rx_user_buffer[LWS_SEND_BUFFER_PRE_PADDING +
 			       (wsi->u.ws.rx_user_buffer_head++)] = c;
@@ -230,7 +234,7 @@ int libwebsocket_client_rx_sm(struct libwebsocket *wsi, unsigned char c)
 			wsi->lws_rx_parse_state = LWS_RXPS_NEW;
 			goto spill;
 		}
-		if (wsi->u.ws.rx_user_buffer_head != MAX_USER_RX_BUFFER)
+		if (wsi->u.ws.rx_user_buffer_head != wsi->protocol->rx_buffer_size)
 			break;
 spill:
 

lib/client.c

@@ -621,6 +621,23 @@ check_accept:
 	/* union transition */
 	memset(&wsi->u, 0, sizeof wsi->u);
 
+	/*
+	 * create the frame buffer for this connection according to the
+	 * size mentioned in the protocol definition.  If 0 there, then
+	 * use a big default for compatibility
+	 */
+
+	n = wsi->protocol->rx_buffer_size;
+	if (!n)
+		n = LWS_MAX_SOCKET_IO_BUF;
+	n += LWS_SEND_BUFFER_PRE_PADDING + LWS_SEND_BUFFER_POST_PADDING;
+	wsi->u.ws.rx_user_buffer = malloc(n);
+	if (!wsi->u.ws.rx_user_buffer) {
+		lwsl_err("Out of Mem allocating rx buffer %d\n", n);
+		goto bail3;
+	}
+	lwsl_info("Allocating client RX buffer %d\n", n);
+
 	lwsl_debug("handshake OK for protocol %s\n", wsi->protocol->name);
 
 	/* call him back to inform him he is up */
@@ -686,8 +703,6 @@ libwebsockets_generate_client_handshake(struct libwebsocket_context *context,
 	struct libwebsocket_extension *ext1;
 	int ext_count = 0;
 #endif
-	unsigned char buf[LWS_SEND_BUFFER_PRE_PADDING + 1 +
-			 MAX_USER_RX_BUFFER + LWS_SEND_BUFFER_POST_PADDING];
 	static const char magic_websocket_guid[] =
 					 "258EAFA5-E914-47DA-95CA-C5AB0DC85B11";
 
@@ -827,10 +842,10 @@ libwebsockets_generate_client_handshake(struct libwebsocket_context *context,
 
 	/* prepare the expected server accept response */
 
-	strcpy((char *)buf, key_b64);
-	strcpy((char *)&buf[strlen((char *)buf)], magic_websocket_guid);
+	strcpy((char *)context->service_buffer, key_b64);
+	strcpy((char *)&context->service_buffer[strlen((char *)context->service_buffer)], magic_websocket_guid);
 
-	SHA1(buf, strlen((char *)buf), (unsigned char *)hash);
+	SHA1(context->service_buffer, strlen((char *)context->service_buffer), (unsigned char *)hash);
 
 	lws_b64_encode_string(hash, 20,
 			wsi->u.hdr.initial_handshake_hash_base64,

lib/extension-deflate-frame.c

@@ -52,8 +52,8 @@ int lws_extension_callback_deflate_frame(
 		}
 		conn->buf_pre_used = 0;
 		conn->buf_pre_length = 0;
-		conn->buf_in_length = MAX_USER_RX_BUFFER;
-		conn->buf_out_length = MAX_USER_RX_BUFFER;
+		conn->buf_in_length = sizeof conn->buf_in;;
+		conn->buf_out_length = sizeof conn->buf_out;
 		conn->compressed_out = 0;
 		conn->buf_pre = NULL;
 		conn->buf_in = (unsigned char *)

lib/extension-deflate-stream.h

@@ -8,8 +8,8 @@ struct lws_ext_deflate_stream_conn {
 	z_stream zs_in;
 	z_stream zs_out;
 	int remaining_in;
-	unsigned char buf_in[MAX_USER_RX_BUFFER];
-	unsigned char buf_out[MAX_USER_RX_BUFFER];
+	unsigned char buf_in[LWS_MAX_SOCKET_IO_BUF];
+	unsigned char buf_out[LWS_MAX_SOCKET_IO_BUF];
 };
 
 extern int lws_extension_callback_deflate_stream(

lib/handshake.c

@@ -212,6 +212,23 @@ libwebsocket_read(struct libwebsocket_context *context,
 		/* union transition */
 		memset(&wsi->u, 0, sizeof wsi->u);
 
+		/*
+		 * create the frame buffer for this connection according to the
+		 * size mentioned in the protocol definition.  If 0 there, use
+		 * a big default for compatibility
+		 */
+
+		n = wsi->protocol->rx_buffer_size;
+		if (!n)
+			n = LWS_MAX_SOCKET_IO_BUF;
+		n += LWS_SEND_BUFFER_PRE_PADDING + LWS_SEND_BUFFER_POST_PADDING;
+		wsi->u.ws.rx_user_buffer = malloc(n);
+		if (!wsi->u.ws.rx_user_buffer) {
+			lwsl_err("Out of Mem allocating rx buffer %d\n", n);
+			goto bail3;
+		}
+		lwsl_info("Allocating client RX buffer %d\n", n);
+
 		lwsl_parser("accepted v%02d connection\n",
 						       wsi->ietf_spec_revision);
 #endif

lib/libwebsockets.c

@@ -312,6 +312,11 @@ just_kill_connection:
 
 	wsi->state = WSI_STATE_DEAD_SOCKET;
 
+	if (old_state == WSI_STATE_ESTABLISHED && wsi->u.ws.rx_user_buffer) {
+		free(wsi->u.ws.rx_user_buffer);
+		wsi->u.ws.rx_user_buffer = NULL;
+	}
+
 	/* tell the user it's all over for this guy */
 
 	if (wsi->protocol && wsi->protocol->callback &&
@@ -1526,7 +1531,6 @@ libwebsocket_create_context(int port, const char *interf,
 	lwsl_info(" LWS_MAX_HEADER_LEN: %u\n", LWS_MAX_HEADER_LEN);
 	lwsl_info(" LWS_INITIAL_HDR_ALLOC: %u\n", LWS_INITIAL_HDR_ALLOC);
 	lwsl_info(" LWS_ADDITIONAL_HDR_ALLOC: %u\n", LWS_ADDITIONAL_HDR_ALLOC);
-	lwsl_info(" MAX_USER_RX_BUFFER: %u\n", MAX_USER_RX_BUFFER);
 	lwsl_info(" LWS_MAX_PROTOCOLS: %u\n", LWS_MAX_PROTOCOLS);
 #ifndef LWS_NO_EXTENSIONS
 	lwsl_info(" LWS_MAX_EXTENSIONS_ACTIVE: %u\n", LWS_MAX_EXTENSIONS_ACTIVE);
@@ -1720,7 +1724,7 @@ libwebsocket_create_context(int port, const char *interf,
 						       "serving unencrypted\n");
 #endif
 
-		lwsl_notice(" per-connection allocation: %u + headers\n", sizeof(struct libwebsocket));
+		lwsl_notice(" per-connection allocation: %u + headers during handshake + frame buffer set by protocol\n", sizeof(struct libwebsocket));
 	}
 #endif
 

lib/libwebsockets.h

@@ -662,6 +662,14 @@ typedef int (extension_callback_function)(struct libwebsocket_context * context,
  *		this much memory allocated on connection establishment and
  *		freed on connection takedown.  A pointer to this per-connection
  *		allocation is passed into the callback in the 'user' parameter
+ * @rx_buffer_size: if you want atomic frames delivered to the callback, you
+ * 		should set this to the size of the biggest legal frame that
+ * 		you support.  If the frame size is exceeded, there is no
+ * 		error, but the buffer will spill to the user callback when
+ * 		full, which you can detect by using
+ * 		libwebsockets_remaining_packet_payload().  Notice that you
+ * 		just talk about frame size here, the LWS_SEND_BUFFER_PRE_PADDING
+ * 		and post-padding are automatically also allocated on top.
  * @owning_server:	the server init call fills in this opaque pointer when
  *		registering this protocol with the server.
  * @protocol_index: which protocol we are starting from zero
@@ -675,6 +683,7 @@ struct libwebsocket_protocols {
 	const char *name;
 	callback_function *callback;
 	size_t per_session_data_size;
+	size_t rx_buffer_size;
 
 	/*
 	 * below are filled in on server init and can be left uninitialized,

lib/parsers.c

@@ -800,6 +800,9 @@ handle_first:
 
 	case LWS_RXPS_PAYLOAD_UNTIL_LENGTH_EXHAUSTED:
 
+		if (!wsi->u.ws.rx_user_buffer)
+			lwsl_err("NULL user buffer...\n");
+
 		if (wsi->u.ws.all_zero_nonce)
 			wsi->u.ws.rx_user_buffer[LWS_SEND_BUFFER_PRE_PADDING +
 			       (wsi->u.ws.rx_user_buffer_head++)] = c;
@@ -812,7 +815,7 @@ handle_first:
 			wsi->lws_rx_parse_state = LWS_RXPS_NEW;
 			goto spill;
 		}
-		if (wsi->u.ws.rx_user_buffer_head != MAX_USER_RX_BUFFER)
+		if (wsi->u.ws.rx_user_buffer_head != wsi->protocol->rx_buffer_size)
 			break;
 spill:
 		/*

lib/private-libwebsockets.h

@@ -125,9 +125,6 @@ SHA1(const unsigned char *d, size_t n, unsigned char *md);
 #ifndef LWS_ADDITIONAL_HDR_ALLOC
 #define LWS_ADDITIONAL_HDR_ALLOC 64
 #endif
-#ifndef MAX_USER_RX_BUFFER
-#define MAX_USER_RX_BUFFER 4096
-#endif
 #ifndef LWS_MAX_PROTOCOLS
 #define LWS_MAX_PROTOCOLS 10
 #endif
@@ -148,6 +145,7 @@ SHA1(const unsigned char *d, size_t n, unsigned char *md);
 #endif
 
 #define MAX_WEBSOCKET_04_KEY_LEN 128
+#define LWS_MAX_SOCKET_IO_BUF 4096
 
 #ifndef SYSTEM_RANDOM_FILEPATH
 #define SYSTEM_RANDOM_FILEPATH "/dev/urandom"
@@ -257,7 +255,7 @@ struct libwebsocket_context {
 	 * does not last longer than the service action (since next service
 	 * of any socket can likewise use it and overwrite)
 	 */
-	unsigned char service_buffer[4096];
+	unsigned char service_buffer[LWS_MAX_SOCKET_IO_BUF];
 
 	int started_with_parent;
 
@@ -324,8 +322,7 @@ struct _lws_header_related {
 };
 
 struct _lws_websocket_related {
-	char rx_user_buffer[LWS_SEND_BUFFER_PRE_PADDING + MAX_USER_RX_BUFFER +
-						  LWS_SEND_BUFFER_POST_PADDING];
+	char *rx_user_buffer;
 	int rx_user_buffer_head;
 	unsigned char masking_key_04[20];
 	unsigned char frame_masking_nonce_04[4];

lib/server-handshake.c

@@ -257,7 +257,6 @@ handshake_0405(struct libwebsocket_context *context, struct libwebsocket *wsi)
 	free(response);
 	wsi->state = WSI_STATE_ESTABLISHED;
 	wsi->lws_rx_parse_state = LWS_RXPS_NEW;
-	wsi->u.ws.rx_packet_length = 0;
 
 	/* notify user code that we're ready to roll */
 

lib/server.c

@@ -127,7 +127,7 @@ int lws_server_socket_service(struct libwebsocket_context *context,
 			struct libwebsocket *wsi, struct pollfd *pollfd)
 {
 	unsigned char buf[LWS_SEND_BUFFER_PRE_PADDING + 1 +
-			 MAX_USER_RX_BUFFER + LWS_SEND_BUFFER_POST_PADDING];
+	                  LWS_MAX_SOCKET_IO_BUF + LWS_SEND_BUFFER_POST_PADDING];
 	struct libwebsocket *new_wsi;
 	int accept_fd;
 	unsigned int clilen;

libwebsockets-api-doc.html

@@ -934,6 +934,7 @@ set the lws_tokens token pointer to it.
 &nbsp; &nbsp; <i>const char *</i> <b>name</b>;<br>
 &nbsp; &nbsp; <i>callback_function *</i> <b>callback</b>;<br>
 &nbsp; &nbsp; <i>size_t</i> <b>per_session_data_size</b>;<br>
+&nbsp; &nbsp; <i>size_t</i> <b>rx_buffer_size</b>;<br>
 &nbsp; &nbsp; <i>struct libwebsocket_context *</i> <b>owning_server</b>;<br>
 &nbsp; &nbsp; <i>int</i> <b>protocol_index</b>;<br>
 };<br>
@@ -951,6 +952,15 @@ the protocol-specific callback
 this much memory allocated on connection establishment and
 freed on connection takedown.  A pointer to this per-connection
 allocation is passed into the callback in the 'user' parameter
+<dt><b>rx_buffer_size</b>
+<dd>if you want atomic frames delivered to the callback, you
+should set this to the size of the biggest legal frame that
+you support.  If the frame size is exceeded, there is no
+error, but the buffer will spill to the user callback when
+full, which you can detect by using
+<b>libwebsockets_remaining_packet_payload</b>.  Notice that you
+just talk about frame size here, the LWS_SEND_BUFFER_PRE_PADDING
+and post-padding are automatically also allocated on top.
 <dt><b>owning_server</b>
 <dd>the server init call fills in this opaque pointer when
 registering this protocol with the server.

test-server/test-client.c

@@ -166,17 +166,15 @@ static struct libwebsocket_protocols protocols[] = {
 		"dumb-increment-protocol",
 		callback_dumb_increment,
 		0,
+		20,
 	},
 	{
 		"lws-mirror-protocol",
 		callback_lws_mirror,
 		0,
+		4096,
 	},
-	{  /* end of list */
-		NULL,
-		NULL,
-		0
-	}
+	{ NULL, NULL, 0, 0 } /* end */
 };
 
 static struct option options[] = {

test-server/test-server.c

@@ -453,21 +453,22 @@ static struct libwebsocket_protocols protocols[] = {
 	{
 		"http-only",		/* name */
 		callback_http,		/* callback */
-		0			/* per_session_data_size */
+		0,			/* per_session_data_size */
+		0,			/* max frame size / rx buffer */
 	},
 	{
 		"dumb-increment-protocol",
 		callback_dumb_increment,
 		sizeof(struct per_session_data__dumb_increment),
+		10,
 	},
 	{
 		"lws-mirror-protocol",
 		callback_lws_mirror,
-		sizeof(struct per_session_data__lws_mirror)
+		sizeof(struct per_session_data__lws_mirror),
+		4096,
 	},
-	{
-		NULL, NULL, 0		/* End of list */
-	}
+	{ NULL, NULL, 0, 0 } /* terminator */
 };
 
 void sighandler(int sig)