http: compression methods
Add generic http compression layer eanbled at cmake with LWS_WITH_HTTP_STREAM_COMPRESSION.
This is wholly a feature of the HTTP role (used by h1 and h2 roles) and doesn't exist
outside that context.
Currently provides 'deflate' and 'br' compression methods for server side only.
'br' requires also -DLWS_WITH_HTTP_BROTLI=1 at cmake and the brotli libraries (available in
your distro already) and dev package.
Other compression methods can be added nicely using an ops struct.
The built-in file serving stuff will use this is the client says he can handle it, and the
mimetype of the file either starts with "text/" (html and css etc) or is the mimetype of
Javascript.
zlib allocates quite a bit while in use, it seems to be around 256KiB per stream. So this
is only useful on relatively strong servers with lots of memory. However for some usecases
where you are serving a lot of css and js assets, it's a nice help.
The patch performs special treatment for http/1.1 pipelining, since the compression is
performed on the fly the compressed content-length is not known until the end. So for h1
only, chunked transfer-encoding is automatically added so pipelining can continue of the
connection.
For h2 the chunking is neither supported nor required, so it "just works".
User code can also request to add a compression transform before the reply headers were
sent using the new api
LWS_VISIBLE int
lws_http_compression_apply(struct lws *wsi, const char *name,
unsigned char **p, unsigned char *end, char decomp);
... this allows transparent compression of dynamically generated HTTP. The requested
compression (eg, "deflate") is only applied if the client headers indicated it was
supported, otherwise it's a NOP.
Name may be NULL in which case the first compression method in the internal table at
stream.c that is mentioned as acceptable by the client will be used.
NOTE: the compression translation, same as h2 support, relies on the user code using
LWS_WRITE_HTTP and then LWS_WRITE_HTTP_FINAL on the last part written. The internal
lws fileserving code already does this.
2018-09-02 14:43:05 +08:00
|
|
|
/*
|
|
|
|
* libwebsockets - small server side websockets and web server implementation
|
|
|
|
*
|
|
|
|
* Copyright (C) 2010 - 2018 Andy Green <andy@warmcat.com>
|
|
|
|
*
|
|
|
|
* This library is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
|
|
* License as published by the Free Software Foundation:
|
|
|
|
* version 2.1 of the License.
|
|
|
|
*
|
|
|
|
* This library is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
* Lesser General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
|
|
* License along with this library; if not, write to the Free Software
|
|
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
|
|
|
* MA 02110-1301 USA
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "core/private.h"
|
|
|
|
|
|
|
|
/* compression methods listed in order of preference */
|
|
|
|
|
|
|
|
struct lws_compression_support *lcs_available[] = {
|
|
|
|
#if defined(LWS_WITH_HTTP_BROTLI)
|
|
|
|
&lcs_brotli,
|
|
|
|
#endif
|
|
|
|
&lcs_deflate,
|
|
|
|
};
|
|
|
|
|
|
|
|
/* compute acceptable compression encodings while we still have an ah */
|
|
|
|
|
|
|
|
int
|
|
|
|
lws_http_compression_validate(struct lws *wsi)
|
|
|
|
{
|
|
|
|
const char *a;
|
|
|
|
size_t n;
|
|
|
|
|
|
|
|
wsi->http.comp_accept_mask = 0;
|
|
|
|
|
|
|
|
if (!wsi->http.ah || !lwsi_role_server(wsi))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
a = lws_hdr_simple_ptr(wsi, WSI_TOKEN_HTTP_ACCEPT_ENCODING);
|
|
|
|
if (!a)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
for (n = 0; n < LWS_ARRAY_SIZE(lcs_available); n++)
|
|
|
|
if (strstr(a, lcs_available[n]->encoding_name))
|
|
|
|
wsi->http.comp_accept_mask |= 1 << n;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
LWS_VISIBLE int
|
|
|
|
lws_http_compression_apply(struct lws *wsi, const char *name,
|
|
|
|
unsigned char **p, unsigned char *end, char decomp)
|
|
|
|
{
|
|
|
|
size_t n;
|
|
|
|
|
|
|
|
for (n = 0; n < LWS_ARRAY_SIZE(lcs_available); n++) {
|
|
|
|
/* if name is non-NULL, choose only that compression method */
|
|
|
|
if (name && !strcmp(lcs_available[n]->encoding_name, name))
|
|
|
|
continue;
|
|
|
|
/*
|
|
|
|
* If we're the server, confirm that the client told us he could
|
|
|
|
* handle this kind of compression transform...
|
|
|
|
*/
|
|
|
|
if (!decomp && !(wsi->http.comp_accept_mask & (1 << n)))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* let's go with this one then... */
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (n == LWS_ARRAY_SIZE(lcs_available))
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
lcs_available[n]->init_compression(&wsi->http.comp_ctx, decomp);
|
|
|
|
if (!wsi->http.comp_ctx.u.generic_ctx_ptr) {
|
|
|
|
lwsl_err("%s: init_compression %d failed\n", __func__, (int)n);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
wsi->http.lcs = lcs_available[n];
|
|
|
|
wsi->http.comp_ctx.may_have_more = 0;
|
|
|
|
wsi->http.comp_ctx.final_on_input_side = 0;
|
|
|
|
wsi->http.comp_ctx.chunking = 0;
|
|
|
|
wsi->http.comp_ctx.is_decompression = decomp;
|
|
|
|
|
|
|
|
if (lws_add_http_header_by_token(wsi, WSI_TOKEN_HTTP_CONTENT_ENCODING,
|
|
|
|
(unsigned char *)lcs_available[n]->encoding_name,
|
|
|
|
strlen(lcs_available[n]->encoding_name), p, end))
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
lwsl_info("%s: wsi %p: applied %s content-encoding\n", __func__,
|
|
|
|
wsi, lcs_available[n]->encoding_name);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
lws_http_compression_destroy(struct lws *wsi)
|
|
|
|
{
|
|
|
|
if (!wsi->http.lcs || !wsi->http.comp_ctx.u.generic_ctx_ptr)
|
|
|
|
return;
|
|
|
|
|
|
|
|
wsi->http.lcs->destroy(&wsi->http.comp_ctx);
|
|
|
|
|
|
|
|
wsi->http.lcs = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This manages the compression transform independent of h1 or h2.
|
|
|
|
*
|
|
|
|
* wsi->buflist_comp stashes pre-transform input that was not yet compressed
|
|
|
|
*/
|
|
|
|
|
|
|
|
int
|
|
|
|
lws_http_compression_transform(struct lws *wsi, unsigned char *buf,
|
|
|
|
size_t len, enum lws_write_protocol *wp,
|
|
|
|
unsigned char **outbuf, size_t *olen_oused)
|
|
|
|
{
|
|
|
|
size_t ilen_iused = len;
|
|
|
|
int n, use = 0, wp1f = (*wp) & 0x1f;
|
|
|
|
lws_comp_ctx_t *ctx = &wsi->http.comp_ctx;
|
|
|
|
|
|
|
|
ctx->may_have_more = 0;
|
|
|
|
|
|
|
|
if (!wsi->http.lcs ||
|
|
|
|
(wp1f != LWS_WRITE_HTTP && wp1f != LWS_WRITE_HTTP_FINAL)) {
|
|
|
|
*outbuf = buf;
|
|
|
|
*olen_oused = len;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (wp1f == LWS_WRITE_HTTP_FINAL) {
|
|
|
|
/*
|
|
|
|
* ...we may get a large buffer that represents the final input
|
|
|
|
* buffer, but it may form multiple frames after being
|
|
|
|
* tranformed by compression; only the last of those is actually
|
|
|
|
* the final frame on the output stream.
|
|
|
|
*
|
|
|
|
* Note that we have received the FINAL input, and downgrade it
|
|
|
|
* to a non-final for now.
|
|
|
|
*/
|
|
|
|
ctx->final_on_input_side = 1;
|
|
|
|
*wp = LWS_WRITE_HTTP | ((*wp) & ~0x1f);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (ctx->buflist_comp || ctx->may_have_more) {
|
|
|
|
/*
|
|
|
|
* we can't send this new stuff when we have old stuff
|
|
|
|
* buffered and not compressed yet. Add it to the tail
|
|
|
|
* and switch to trying to process the head.
|
|
|
|
*/
|
|
|
|
if (buf && len) {
|
2019-07-13 12:06:33 -07:00
|
|
|
if (lws_buflist_append_segment(
|
|
|
|
&ctx->buflist_comp, buf, len) < 0)
|
|
|
|
return -1;
|
http: compression methods
Add generic http compression layer eanbled at cmake with LWS_WITH_HTTP_STREAM_COMPRESSION.
This is wholly a feature of the HTTP role (used by h1 and h2 roles) and doesn't exist
outside that context.
Currently provides 'deflate' and 'br' compression methods for server side only.
'br' requires also -DLWS_WITH_HTTP_BROTLI=1 at cmake and the brotli libraries (available in
your distro already) and dev package.
Other compression methods can be added nicely using an ops struct.
The built-in file serving stuff will use this is the client says he can handle it, and the
mimetype of the file either starts with "text/" (html and css etc) or is the mimetype of
Javascript.
zlib allocates quite a bit while in use, it seems to be around 256KiB per stream. So this
is only useful on relatively strong servers with lots of memory. However for some usecases
where you are serving a lot of css and js assets, it's a nice help.
The patch performs special treatment for http/1.1 pipelining, since the compression is
performed on the fly the compressed content-length is not known until the end. So for h1
only, chunked transfer-encoding is automatically added so pipelining can continue of the
connection.
For h2 the chunking is neither supported nor required, so it "just works".
User code can also request to add a compression transform before the reply headers were
sent using the new api
LWS_VISIBLE int
lws_http_compression_apply(struct lws *wsi, const char *name,
unsigned char **p, unsigned char *end, char decomp);
... this allows transparent compression of dynamically generated HTTP. The requested
compression (eg, "deflate") is only applied if the client headers indicated it was
supported, otherwise it's a NOP.
Name may be NULL in which case the first compression method in the internal table at
stream.c that is mentioned as acceptable by the client will be used.
NOTE: the compression translation, same as h2 support, relies on the user code using
LWS_WRITE_HTTP and then LWS_WRITE_HTTP_FINAL on the last part written. The internal
lws fileserving code already does this.
2018-09-02 14:43:05 +08:00
|
|
|
lwsl_debug("%s: %p: adding %d to comp buflist\n",
|
|
|
|
__func__,wsi, (int)len);
|
|
|
|
}
|
|
|
|
|
|
|
|
len = lws_buflist_next_segment_len(&ctx->buflist_comp, &buf);
|
|
|
|
ilen_iused = len;
|
|
|
|
use = 1;
|
|
|
|
lwsl_debug("%s: %p: trying comp buflist %d\n", __func__, wsi,
|
|
|
|
(int)len);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!buf && ilen_iused)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
lwsl_debug("%s: %p: pre-process: ilen_iused %d, olen_oused %d\n",
|
|
|
|
__func__, wsi, (int)ilen_iused, (int)*olen_oused);
|
|
|
|
|
|
|
|
n = wsi->http.lcs->process(ctx, buf, &ilen_iused, *outbuf, olen_oused);
|
|
|
|
|
|
|
|
if (n && n != 1) {
|
|
|
|
lwsl_err("%s: problem with compression\n", __func__);
|
|
|
|
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!ctx->may_have_more && ctx->final_on_input_side)
|
|
|
|
*wp = LWS_WRITE_HTTP_FINAL | ((*wp) & ~0x1f);
|
|
|
|
|
|
|
|
lwsl_debug("%s: %p: more %d, ilen_iused %d\n", __func__, wsi,
|
|
|
|
ctx->may_have_more, (int)ilen_iused);
|
|
|
|
|
|
|
|
if (use && ilen_iused) {
|
|
|
|
/*
|
|
|
|
* we were flushing stuff from the buflist head... account for
|
|
|
|
* however much actually got processed by the compression
|
|
|
|
* transform
|
|
|
|
*/
|
|
|
|
lws_buflist_use_segment(&ctx->buflist_comp, ilen_iused);
|
|
|
|
lwsl_debug("%s: %p: marking %d of comp buflist as used "
|
|
|
|
"(ctx->buflist_comp %p)\n", __func__, wsi,
|
|
|
|
(int)len, ctx->buflist_comp);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!use && ilen_iused != len) {
|
|
|
|
/*
|
|
|
|
* ...we were sending stuff from the caller directly and not
|
|
|
|
* all of it got processed... stash on the buflist tail
|
|
|
|
*/
|
2019-07-13 12:06:33 -07:00
|
|
|
if (lws_buflist_append_segment(&ctx->buflist_comp,
|
|
|
|
buf + ilen_iused, len - ilen_iused) < 0)
|
|
|
|
return -1;
|
http: compression methods
Add generic http compression layer eanbled at cmake with LWS_WITH_HTTP_STREAM_COMPRESSION.
This is wholly a feature of the HTTP role (used by h1 and h2 roles) and doesn't exist
outside that context.
Currently provides 'deflate' and 'br' compression methods for server side only.
'br' requires also -DLWS_WITH_HTTP_BROTLI=1 at cmake and the brotli libraries (available in
your distro already) and dev package.
Other compression methods can be added nicely using an ops struct.
The built-in file serving stuff will use this is the client says he can handle it, and the
mimetype of the file either starts with "text/" (html and css etc) or is the mimetype of
Javascript.
zlib allocates quite a bit while in use, it seems to be around 256KiB per stream. So this
is only useful on relatively strong servers with lots of memory. However for some usecases
where you are serving a lot of css and js assets, it's a nice help.
The patch performs special treatment for http/1.1 pipelining, since the compression is
performed on the fly the compressed content-length is not known until the end. So for h1
only, chunked transfer-encoding is automatically added so pipelining can continue of the
connection.
For h2 the chunking is neither supported nor required, so it "just works".
User code can also request to add a compression transform before the reply headers were
sent using the new api
LWS_VISIBLE int
lws_http_compression_apply(struct lws *wsi, const char *name,
unsigned char **p, unsigned char *end, char decomp);
... this allows transparent compression of dynamically generated HTTP. The requested
compression (eg, "deflate") is only applied if the client headers indicated it was
supported, otherwise it's a NOP.
Name may be NULL in which case the first compression method in the internal table at
stream.c that is mentioned as acceptable by the client will be used.
NOTE: the compression translation, same as h2 support, relies on the user code using
LWS_WRITE_HTTP and then LWS_WRITE_HTTP_FINAL on the last part written. The internal
lws fileserving code already does this.
2018-09-02 14:43:05 +08:00
|
|
|
|
|
|
|
lwsl_debug("%s: buffering %d unused comp input\n", __func__,
|
|
|
|
(int)(len - ilen_iused));
|
|
|
|
}
|
|
|
|
if (ctx->buflist_comp || ctx->may_have_more)
|
|
|
|
lws_callback_on_writable(wsi);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|