From d58828692e6832a2878df9375d39289df5144fa0 Mon Sep 17 00:00:00 2001 From: Andy Green Date: Sun, 2 Sep 2018 14:43:05 +0800 Subject: [PATCH] http: compression methods Add generic http compression layer eanbled at cmake with LWS_WITH_HTTP_STREAM_COMPRESSION. This is wholly a feature of the HTTP role (used by h1 and h2 roles) and doesn't exist outside that context. Currently provides 'deflate' and 'br' compression methods for server side only. 'br' requires also -DLWS_WITH_HTTP_BROTLI=1 at cmake and the brotli libraries (available in your distro already) and dev package. Other compression methods can be added nicely using an ops struct. The built-in file serving stuff will use this is the client says he can handle it, and the mimetype of the file either starts with "text/" (html and css etc) or is the mimetype of Javascript. zlib allocates quite a bit while in use, it seems to be around 256KiB per stream. So this is only useful on relatively strong servers with lots of memory. However for some usecases where you are serving a lot of css and js assets, it's a nice help. The patch performs special treatment for http/1.1 pipelining, since the compression is performed on the fly the compressed content-length is not known until the end. So for h1 only, chunked transfer-encoding is automatically added so pipelining can continue of the connection. For h2 the chunking is neither supported nor required, so it "just works". User code can also request to add a compression transform before the reply headers were sent using the new api LWS_VISIBLE int lws_http_compression_apply(struct lws *wsi, const char *name, unsigned char **p, unsigned char *end, char decomp); ... this allows transparent compression of dynamically generated HTTP. The requested compression (eg, "deflate") is only applied if the client headers indicated it was supported, otherwise it's a NOP. Name may be NULL in which case the first compression method in the internal table at stream.c that is mentioned as acceptable by the client will be used. NOTE: the compression translation, same as h2 support, relies on the user code using LWS_WRITE_HTTP and then LWS_WRITE_HTTP_FINAL on the last part written. The internal lws fileserving code already does this. --- CMakeLists.txt | 25 ++- cmake/lws_config.h.in | 3 + lib/core/libwebsockets.c | 14 +- lib/core/output.c | 17 +- lib/core/service.c | 24 ++ lib/libwebsockets.h | 29 +++ lib/plat/esp32/esp32-sockets.c | 7 +- lib/plat/optee/lws-plat-optee.c | 7 +- lib/plat/unix/unix-sockets.c | 7 +- lib/plat/windows/windows-sockets.c | 7 +- lib/roles/h1/ops-h1.c | 158 ++++++++++++- lib/roles/h2/ops-h2.c | 92 +++++++- lib/roles/http/compression/README.md | 17 ++ lib/roles/http/compression/brotli/brotli.c | 122 ++++++++++ lib/roles/http/compression/deflate/deflate.c | 110 +++++++++ lib/roles/http/compression/private.h | 80 +++++++ lib/roles/http/compression/stream.c | 221 +++++++++++++++++++ lib/roles/http/header.c | 17 ++ lib/roles/http/private.h | 11 + lib/roles/http/server/parsers.c | 6 + lib/roles/http/server/server.c | 102 ++++++++- lib/tls/mbedtls/ssl.c | 2 +- 22 files changed, 1046 insertions(+), 32 deletions(-) create mode 100644 lib/roles/http/compression/README.md create mode 100644 lib/roles/http/compression/brotli/brotli.c create mode 100644 lib/roles/http/compression/deflate/deflate.c create mode 100644 lib/roles/http/compression/private.h create mode 100644 lib/roles/http/compression/stream.c diff --git a/CMakeLists.txt b/CMakeLists.txt index edd94a76d..1717102df 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,8 @@ option(LWS_WITH_PEER_LIMITS "Track peers and restrict resources a single peer ca option(LWS_WITH_ACCESS_LOG "Support generating Apache-compatible access logs" OFF) option(LWS_WITH_RANGES "Support http ranges (RFC7233)" OFF) option(LWS_WITH_SERVER_STATUS "Support json + jscript server monitoring" OFF) +option(LWS_WITH_HTTP_STREAM_COMPRESSION "Support HTTP stream compression" OFF) +option(LWS_WITH_HTTP_BROTLI "Also offer brotli http stream compression (requires LWS_WITH_HTTP_STREAM_COMPRESSION)" OFF) option(LWS_WITH_ACME "Enable support for ACME automatic cert acquisition + maintenance (letsencrypt etc)" OFF) # # TLS library options... all except mbedTLS are basically OpenSSL variants. @@ -438,6 +440,10 @@ if (LWS_WITH_SSL AND LWS_WITH_MBEDTLS) set(USE_MBEDTLS 1) endif() +if (LWS_WITH_HTTP_STREAM_COMPRESSION) + set(LWS_WITH_ZLIB 1) +endif() + if (LWS_WITH_ZLIB AND NOT LWS_WITH_BUNDLED_ZLIB) if ("${LWS_ZLIB_LIBRARIES}" STREQUAL "" OR "${LWS_ZLIB_INCLUDE_DIRS}" STREQUAL "") else() @@ -746,6 +752,15 @@ if (LWS_ROLE_H1 OR LWS_ROLE_H2) list(APPEND SOURCES lib/roles/http/header.c lib/roles/http/server/parsers.c) + if (LWS_WITH_HTTP_STREAM_COMPRESSION) + list(APPEND SOURCES + lib/roles/http/compression/stream.c + lib/roles/http/compression/deflate/deflate.c) + if (LWS_WITH_HTTP_BROTLI) + list(APPEND SOURCES + lib/roles/http/compression/brotli/brotli.c) + endif() + endif() endif() if (LWS_ROLE_H1) @@ -1201,7 +1216,7 @@ set(LIB_LIST) # # -# ZLIB (Only needed for deflate extensions). +# ZLIB (needed for deflate extension and if LWS_WITH_HTTP_STREAM_COMPRESSION) # if (LWS_WITH_ZLIB) if (LWS_WITH_BUNDLED_ZLIB) @@ -1242,6 +1257,10 @@ if (LWS_WITH_ZLIB) list(APPEND LIB_LIST ${ZLIB_LIBRARIES}) endif() +if (LWS_WITH_HTTP_BROTLI) + list(APPEND LIB_LIST brotlienc brotlidec brotlidec) +endif() + # # OpenSSL # @@ -1620,6 +1639,10 @@ if ((LWS_ROLE_H1 OR LWS_ROLE_H2) AND NOT LWS_WITHOUT_TESTAPPS) add_dependencies(${TEST_NAME} websockets) endif() + if (LWS_WITH_HTTP_STREAM_COMPRESSION) + target_link_libraries(${TEST_NAME} z) + endif() + # Set test app specific defines. set_property(TARGET ${TEST_NAME} PROPERTY COMPILE_DEFINITIONS diff --git a/cmake/lws_config.h.in b/cmake/lws_config.h.in index ea5367c9e..6c6418c04 100644 --- a/cmake/lws_config.h.in +++ b/cmake/lws_config.h.in @@ -180,4 +180,7 @@ #cmakedefine LWS_HAS_INTPTR_T +#cmakedefine LWS_WITH_HTTP_STREAM_COMPRESSION +#cmakedefine LWS_WITH_HTTP_BROTLI + ${LWS_SIZEOFPTR_CODE} diff --git a/lib/core/libwebsockets.c b/lib/core/libwebsockets.c index 10ecb4724..eeb9e168f 100644 --- a/lib/core/libwebsockets.c +++ b/lib/core/libwebsockets.c @@ -723,14 +723,24 @@ __lws_close_free_wsi(struct lws *wsi, enum lws_close_status reason, const char * goto just_kill_connection; case LRS_FLUSHING_BEFORE_CLOSE: - if (lws_has_buffered_out(wsi)) { + if (lws_has_buffered_out(wsi) +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + || wsi->http.comp_ctx.buflist_comp || + wsi->http.comp_ctx.may_have_more +#endif + ) { lws_callback_on_writable(wsi); return; } lwsl_info("%p: end LRS_FLUSHING_BEFORE_CLOSE\n", wsi); goto just_kill_connection; default: - if (lws_has_buffered_out(wsi)) { + if (lws_has_buffered_out(wsi) +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + || wsi->http.comp_ctx.buflist_comp || + wsi->http.comp_ctx.may_have_more +#endif + ) { lwsl_info("%p: LRS_FLUSHING_BEFORE_CLOSE\n", wsi); lwsi_set_state(wsi, LRS_FLUSHING_BEFORE_CLOSE); __lws_set_timeout(wsi, diff --git a/lib/core/output.c b/lib/core/output.c index 3e0ffe43d..8f6349bc1 100644 --- a/lib/core/output.c +++ b/lib/core/output.c @@ -31,12 +31,14 @@ int lws_issue_raw(struct lws *wsi, unsigned char *buf, size_t len) size_t real_len = len; unsigned int n; + // lwsl_notice("%s: len %d\n", __func__, (int)len); + /* * Detect if we got called twice without going through the * event loop to handle pending. Since that guarantees extending any * existing buflist_out it's inefficient. */ - if (buf && wsi->could_have_pending) { + if (0 && buf && wsi->could_have_pending) { lwsl_hexdump_level(LLL_INFO, buf, len); lwsl_info("** %p: vh: %s, prot: %s, role %s: " "Inefficient back-to-back write of %lu detected...\n", @@ -49,7 +51,11 @@ int lws_issue_raw(struct lws *wsi, unsigned char *buf, size_t len) /* just ignore sends after we cleared the truncation buffer */ if (lwsi_state(wsi) == LRS_FLUSHING_BEFORE_CLOSE && - !lws_has_buffered_out(wsi)) + !lws_has_buffered_out(wsi) +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + && !wsi->http.comp_ctx.may_have_more +#endif + ) return (int)len; if (buf && lws_has_buffered_out(wsi)) { @@ -74,6 +80,8 @@ int lws_issue_raw(struct lws *wsi, unsigned char *buf, size_t len) len = lws_buflist_next_segment_len(&wsi->buflist_out, &buf); real_len = len; + + lwsl_debug("%s: draining %d\n", __func__, (int)len); } if (!len) @@ -155,6 +163,11 @@ int lws_issue_raw(struct lws *wsi, unsigned char *buf, size_t len) return n; } +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + if (wsi->http.comp_ctx.may_have_more) + lws_callback_on_writable(wsi); +#endif + if ((unsigned int)n == real_len) /* what we just sent went out cleanly */ return n; diff --git a/lib/core/service.c b/lib/core/service.c index 994b16998..3c4b8f106 100644 --- a/lib/core/service.c +++ b/lib/core/service.c @@ -74,6 +74,8 @@ lws_handle_POLLOUT_event(struct lws *wsi, struct lws_pollfd *pollfd) * Priority 1: pending truncated sends are incomplete ws fragments * If anything else sent first the protocol would be * corrupted. + * + * These are post- any compression transform */ if (lws_has_buffered_out(wsi)) { @@ -90,6 +92,28 @@ lws_handle_POLLOUT_event(struct lws *wsi, struct lws_pollfd *pollfd) goto bail_die; /* retry closing now */ } + /* Priority 2: pre- compression transform */ + +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + if (wsi->http.comp_ctx.buflist_comp || + wsi->http.comp_ctx.may_have_more) { + enum lws_write_protocol wp = LWS_WRITE_HTTP; + + lwsl_debug("%s: completing comp partial (buflist_comp %p, may %d)\n", + __func__, wsi->http.comp_ctx.buflist_comp, + wsi->http.comp_ctx.may_have_more + ); + + if (wsi->role_ops->write_role_protocol(wsi, NULL, 0, &wp) < 0) { + lwsl_info("%s signalling to close\n", __func__); + goto bail_die; + } + lws_callback_on_writable(wsi); + + goto bail_ok; + } +#endif + #ifdef LWS_WITH_CGI /* * A cgi master's wire protocol remains h1 or h2. He is just getting diff --git a/lib/libwebsockets.h b/lib/libwebsockets.h index 65de30ecb..74a414ad2 100644 --- a/lib/libwebsockets.h +++ b/lib/libwebsockets.h @@ -3410,6 +3410,35 @@ struct lws_http_mount { void *_unused[2]; /**< dummy */ }; + +/** + * lws_http_compression_apply() - apply an http compression transform + * + * \param wsi: the wsi to apply the compression transform to + * \param name: NULL, or the name of the compression transform, eg, "deflate" + * \param p: pointer to pointer to headers buffer + * \param end: pointer to end of headers buffer + * \param decomp: 0 = add compressor to wsi, 1 = add decompressor + * + * This allows transparent compression of dynamically generated HTTP. The + * requested compression (eg, "deflate") is only applied if the client headers + * indicated it was supported (and it has support in lws), otherwise it's a NOP. + * + * If the requested compression method is NULL, then the supported compression + * formats are tried, and for non-decompression (server) mode the first that's + * found on the client's accept-encoding header is chosen. + * + * NOTE: the compression transform, same as h2 support, relies on the user + * code using LWS_WRITE_HTTP and then LWS_WRITE_HTTP_FINAL on the last part + * written. The internal lws fileserving code already does this. + * + * If the library was built without the cmake option + * LWS_WITH_HTTP_STREAM_COMPRESSION set, then a NOP is provided for this api, + * allowing user code to build either way and use compression if available. + */ +LWS_VISIBLE int +lws_http_compression_apply(struct lws *wsi, const char *name, + unsigned char **p, unsigned char *end, char decomp); ///@} ///@} diff --git a/lib/plat/esp32/esp32-sockets.c b/lib/plat/esp32/esp32-sockets.c index 9bcaf3bd3..67b39a824 100644 --- a/lib/plat/esp32/esp32-sockets.c +++ b/lib/plat/esp32/esp32-sockets.c @@ -44,7 +44,12 @@ lws_send_pipe_choked(struct lws *wsi) wsi_eff->could_have_pending = 0; /* treat the fact we got a truncated send pending as if we're choked */ - if (lws_has_buffered_out(wsi)) + if (lws_has_buffered_out(wsi) +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + || wsi->http.comp_ctx.buflist_comp || + wsi->http.comp_ctx.may_have_more +#endif + ) return 1; FD_ZERO(&writefds); diff --git a/lib/plat/optee/lws-plat-optee.c b/lib/plat/optee/lws-plat-optee.c index 6d5e32b39..f1000182c 100644 --- a/lib/plat/optee/lws-plat-optee.c +++ b/lib/plat/optee/lws-plat-optee.c @@ -54,7 +54,12 @@ lws_send_pipe_choked(struct lws *wsi) wsi_eff->could_have_pending = 0; /* treat the fact we got a truncated send pending as if we're choked */ - if (lws_has_buffered_out(wsi_eff)) + if (lws_has_buffered_out(wsi_eff) +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + || wsi->http.comp_ctx.buflist_comp || + wsi->http.comp_ctx.may_have_more +#endif + ) return 1; #if 0 diff --git a/lib/plat/unix/unix-sockets.c b/lib/plat/unix/unix-sockets.c index e8b734aab..b423eaa1e 100644 --- a/lib/plat/unix/unix-sockets.c +++ b/lib/plat/unix/unix-sockets.c @@ -41,7 +41,12 @@ lws_send_pipe_choked(struct lws *wsi) wsi_eff->could_have_pending = 0; /* treat the fact we got a truncated send pending as if we're choked */ - if (lws_has_buffered_out(wsi_eff)) + if (lws_has_buffered_out(wsi_eff) +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + ||wsi->http.comp_ctx.buflist_comp || + wsi->http.comp_ctx.may_have_more +#endif + ) return 1; fds.fd = wsi_eff->desc.sockfd; diff --git a/lib/plat/windows/windows-sockets.c b/lib/plat/windows/windows-sockets.c index 3a5d72550..99d9f6a01 100644 --- a/lib/plat/windows/windows-sockets.c +++ b/lib/plat/windows/windows-sockets.c @@ -15,7 +15,12 @@ lws_send_pipe_choked(struct lws *wsi) wsi_eff->could_have_pending = 0; /* treat the fact we got a truncated send pending as if we're choked */ - if (lws_has_buffered_out(wsi_eff)) + if (lws_has_buffered_out(wsi_eff) +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + ||wsi->http.comp_ctx.buflist_comp || + wsi->http.comp_ctx.may_have_more +#endif + ) return 1; return (int)wsi_eff->sock_send_blocking; diff --git a/lib/roles/h1/ops-h1.c b/lib/roles/h1/ops-h1.c index 09ff88487..9d53730f9 100644 --- a/lib/roles/h1/ops-h1.c +++ b/lib/roles/h1/ops-h1.c @@ -522,6 +522,27 @@ rops_handle_POLLIN_h1(struct lws_context_per_thread *pt, struct lws *wsi, } #endif +#if 0 + + /* + * !!! lws_serve_http_file_fragment() seems to duplicate most of + * lws_handle_POLLOUT_event() in its own loop... + */ + lwsl_debug("%s: %d %d\n", __func__, (pollfd->revents & LWS_POLLOUT), + lwsi_state_can_handle_POLLOUT(wsi)); + + if ((pollfd->revents & LWS_POLLOUT) && + lwsi_state_can_handle_POLLOUT(wsi) && + lws_handle_POLLOUT_event(wsi, pollfd)) { + if (lwsi_state(wsi) == LRS_RETURNED_CLOSE) + lwsi_set_state(wsi, LRS_FLUSHING_BEFORE_CLOSE); + /* the write failed... it's had it */ + wsi->socket_is_permanently_unusable = 1; + + return LWS_HPI_RET_PLEASE_CLOSE_ME; + } +#endif + if (lws_is_flowcontrolled(wsi)) /* We cannot deal with any kind of new RX because we are * RX-flowcontrolled. @@ -611,19 +632,67 @@ static int rops_write_role_protocol_h1(struct lws *wsi, unsigned char *buf, size_t len, enum lws_write_protocol *wp) { -#if 0 - /* if not in a state to send stuff, then just send nothing */ + size_t olen = len; + int n; - if ((lwsi_state(wsi) != LRS_RETURNED_CLOSE && - lwsi_state(wsi) != LRS_WAITING_TO_SEND_CLOSE && - lwsi_state(wsi) != LRS_AWAITING_CLOSE_ACK)) { - //assert(0); - lwsl_debug("binning %d %d\n", lwsi_state(wsi), *wp); - return 0; +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + if (wsi->http.lcs && (((*wp) & 0x1f) == LWS_WRITE_HTTP_FINAL || + ((*wp) & 0x1f) == LWS_WRITE_HTTP)) { + unsigned char mtubuf[1400 + LWS_PRE + + LWS_HTTP_CHUNK_HDR_MAX_SIZE + + LWS_HTTP_CHUNK_TRL_MAX_SIZE], + *out = mtubuf + LWS_PRE + + LWS_HTTP_CHUNK_HDR_MAX_SIZE; + size_t o = sizeof(mtubuf) - LWS_PRE - + LWS_HTTP_CHUNK_HDR_MAX_SIZE - + LWS_HTTP_CHUNK_TRL_MAX_SIZE; + char c[LWS_HTTP_CHUNK_HDR_MAX_SIZE + 2]; + + n = lws_http_compression_transform(wsi, buf, len, wp, &out, &o); + if (n) + return n; + + lwsl_debug("%s: %p: transformed %d bytes to %d " + "(wp 0x%x, more %d)\n", __func__, wsi, (int)len, + (int)o, (int)*wp, wsi->http.comp_ctx.may_have_more); + + if (!o) + return olen; + + if (wsi->http.comp_ctx.chunking) { + /* + * this only needs dealing with on http/1.1 to allow + * pipelining + */ + n = lws_snprintf(c, sizeof(c), "%X\x0d\x0a", (int)o); + lwsl_notice("%s: chunk %s\n", __func__, c); + out -= n; + o += n; + memcpy(out, c, n); + out[o++] = '\x0d'; + out[o++] = '\x0a'; + + if (((*wp) & 0x1f) == LWS_WRITE_HTTP_FINAL) { + out[o++] = '0'; + out[o++] = '\x0d'; + out[o++] = '\x0a'; + out[o++] = '\x0d'; + out[o++] = '\x0a'; + } + } + + buf = out; + len = o; } #endif - return lws_issue_raw(wsi, (unsigned char *)buf, len); + n = lws_issue_raw(wsi, (unsigned char *)buf, len); + if (n < 0) + return n; + + /* hide there may have been compression */ + + return olen; } static int @@ -669,6 +738,10 @@ rops_destroy_role_h1(struct lws *wsi) ah = ah->next; } +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + lws_http_compression_destroy(wsi); +#endif + #ifdef LWS_ROLE_WS lws_free_set_NULL(wsi->ws); #endif @@ -802,6 +875,73 @@ fail_wsi: } #endif +#if 0 +static int +rops_perform_user_POLLOUT_h1(struct lws *wsi) +{ + volatile struct lws *vwsi = (volatile struct lws *)wsi; + int n; + + /* priority 1: post compression-transform buffered output */ + + if (lws_has_buffered_out(wsi)) { + lwsl_debug("%s: completing partial\n", __func__); + if (lws_issue_raw(wsi, NULL, 0) < 0) { + lwsl_info("%s signalling to close\n", __func__); + return -1; + } + n = 0; + vwsi->leave_pollout_active = 1; + goto cleanup; + } + + /* priority 2: pre compression-transform buffered output */ + +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + if (wsi->http.comp_ctx.buflist_comp || + wsi->http.comp_ctx.may_have_more) { + enum lws_write_protocol wp = LWS_WRITE_HTTP; + + lwsl_debug("%s: completing comp partial" + "(buflist_comp %p, may %d)\n", + __func__, wsi->http.comp_ctx.buflist_comp, + wsi->http.comp_ctx.may_have_more); + + if (rops_write_role_protocol_h1(wsi, NULL, 0, &wp) < 0) { + lwsl_info("%s signalling to close\n", __func__); + lws_close_free_wsi(wsi, LWS_CLOSE_STATUS_NOSTATUS, + "comp write fail"); + } + n = 0; + vwsi->leave_pollout_active = 1; + goto cleanup; + } +#endif + + /* priority 3: if no buffered out and waiting for that... */ + + if (lwsi_state(wsi) == LRS_FLUSHING_BEFORE_CLOSE) { + wsi->socket_is_permanently_unusable = 1; + return -1; + } + + /* priority 4: user writeable callback */ + + vwsi = (volatile struct lws *)wsi; + vwsi->leave_pollout_active = 0; + + n = lws_callback_as_writeable(wsi); + +cleanup: + vwsi->handling_pollout = 0; + + if (vwsi->leave_pollout_active) + lws_change_pollfd(wsi, 0, LWS_POLLOUT); + + return n; +} +#endif + struct lws_role_ops role_ops_h1 = { /* role name */ "h1", /* alpn id */ "http/1.1", diff --git a/lib/roles/h2/ops-h2.c b/lib/roles/h2/ops-h2.c index e93cd6a07..cea49652d 100644 --- a/lib/roles/h2/ops-h2.c +++ b/lib/roles/h2/ops-h2.c @@ -360,6 +360,7 @@ rops_write_role_protocol_h2(struct lws *wsi, unsigned char *buf, size_t len, enum lws_write_protocol *wp) { unsigned char flags = 0, base = (*wp) & 0x1f; + size_t olen = len; int n; /* if not in a state to send stuff, then just send nothing */ @@ -381,6 +382,31 @@ rops_write_role_protocol_h2(struct lws *wsi, unsigned char *buf, size_t len, return 0; } + /* compression transform... */ + +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + if (wsi->http.lcs) { + unsigned char mtubuf[1450 + LWS_PRE], *out = mtubuf + LWS_PRE; + size_t o = sizeof(mtubuf) - LWS_PRE; + + n = lws_http_compression_transform(wsi, buf, len, wp, &out, &o); + if (n) + return n; + + lwsl_debug("%s: %p: transformed %d bytes to %d " + "(wp 0x%x, more %d)\n", __func__, + wsi, (int)len, (int)o, (int)*wp, + wsi->http.comp_ctx.may_have_more); + + buf = out; + len = o; + base = (*wp) & 0x1f; + + if (!len) + return olen; + } +#endif + /* * ws-over-h2 also ends up here after the ws framing applied */ @@ -401,7 +427,8 @@ rops_write_role_protocol_h2(struct lws *wsi, unsigned char *buf, size_t len, n = LWS_H2_FRAME_TYPE_CONTINUATION; if (!((*wp) & LWS_WRITE_NO_FIN)) flags = LWS_H2_FLAG_END_HEADERS; - if (wsi->h2.send_END_STREAM || ((*wp) & LWS_WRITE_H2_STREAM_END)) { + if (wsi->h2.send_END_STREAM || + ((*wp) & LWS_WRITE_H2_STREAM_END)) { flags |= LWS_H2_FLAG_END_STREAM; wsi->h2.send_END_STREAM = 1; } @@ -420,12 +447,18 @@ rops_write_role_protocol_h2(struct lws *wsi, unsigned char *buf, size_t len, } if (base == LWS_WRITE_HTTP_FINAL || ((*wp) & LWS_WRITE_H2_STREAM_END)) { - lwsl_info("%s: setting END_STREAM\n", __func__); + lwsl_info("%s: %p: setting END_STREAM\n", __func__, wsi); flags |= LWS_H2_FLAG_END_STREAM; wsi->h2.send_END_STREAM = 1; } - return lws_h2_frame_write(wsi, n, flags, wsi->h2.my_sid, (int)len, buf); + n = lws_h2_frame_write(wsi, n, flags, wsi->h2.my_sid, (int)len, buf); + if (n < 0) + return n; + + /* hide it may have been compressed... */ + + return olen; } static int @@ -523,6 +556,10 @@ rops_destroy_role_h2(struct lws *wsi) ah = ah->next; } +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + lws_http_compression_destroy(wsi); +#endif + if (wsi->upgraded_to_http2 || wsi->http2_substream) { lws_hpack_destroy_dynamic_header(wsi); @@ -822,6 +859,55 @@ rops_perform_user_POLLOUT_h2(struct lws *wsi) lwsl_info("%s: child %p (wsistate 0x%x)\n", __func__, w, w->wsistate); + /* priority 1: post compression-transform buffered output */ + + if (lws_has_buffered_out(w)) { + lwsl_debug("%s: completing partial\n", __func__); + if (lws_issue_raw(w, NULL, 0) < 0) { + lwsl_info("%s signalling to close\n", __func__); + lws_close_free_wsi(w, LWS_CLOSE_STATUS_NOSTATUS, + "h2 end stream 1"); + wa = &wsi->h2.child_list; + goto next_child; + } + lws_callback_on_writable(w); + wa = &wsi->h2.child_list; + goto next_child; + } + + /* priority 2: pre compression-transform buffered output */ + +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + if (w->http.comp_ctx.buflist_comp || + w->http.comp_ctx.may_have_more) { + enum lws_write_protocol wp = LWS_WRITE_HTTP; + + lwsl_debug("%s: completing comp partial" + "(buflist_comp %p, may %d)\n", + __func__, w->http.comp_ctx.buflist_comp, + w->http.comp_ctx.may_have_more); + + if (rops_write_role_protocol_h2(w, NULL, 0, &wp) < 0) { + lwsl_info("%s signalling to close\n", __func__); + lws_close_free_wsi(w, LWS_CLOSE_STATUS_NOSTATUS, + "comp write fail"); + } + lws_callback_on_writable(w); + wa = &wsi->h2.child_list; + goto next_child; + } +#endif + + /* priority 3: if no buffered out and waiting for that... */ + + if (lwsi_state(w) == LRS_FLUSHING_BEFORE_CLOSE) { + w->socket_is_permanently_unusable = 1; + lws_close_free_wsi(w, LWS_CLOSE_STATUS_NOSTATUS, + "h2 end stream 1"); + wa = &wsi->h2.child_list; + goto next_child; + } + /* if we arrived here, even by looping, we checked choked */ w->could_have_pending = 0; wsi->could_have_pending = 0; diff --git a/lib/roles/http/compression/README.md b/lib/roles/http/compression/README.md new file mode 100644 index 000000000..8d9d57f85 --- /dev/null +++ b/lib/roles/http/compression/README.md @@ -0,0 +1,17 @@ +HTTP compression +---------------- + +This directory contains generic compression transforms that can be applied to +specifically HTTP content streams, after the header, be it h1 or h2. + +The compression transforms expose an "ops" type struct and a compressor name +as used by `content-encoding`... the ops struct definition can be found in +./private.h. + +Because the compression transform depends on being able to send on its output +before it can process new input, the transform adds a new kind of buflist +`wsi->buflist_comp` that represents pre-compression transform data +("input data" from the perspective of the compression transform) that was +delivered to be processed but couldn't be accepted. + +Currently, zlib 'deflate' and brotli 'br' are supported on the server side. diff --git a/lib/roles/http/compression/brotli/brotli.c b/lib/roles/http/compression/brotli/brotli.c new file mode 100644 index 000000000..14ad96104 --- /dev/null +++ b/lib/roles/http/compression/brotli/brotli.c @@ -0,0 +1,122 @@ +/* + * libwebsockets - small server side websockets and web server implementation + * + * Copyright (C) 2010 - 2018 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include "core/private.h" + + +static int +lcs_init_compression_brotli(lws_comp_ctx_t *ctx, int decomp) +{ + ctx->is_decompression = decomp; + + if (!decomp) { + ctx->u.br_en = BrotliEncoderCreateInstance(NULL, NULL, NULL); + if (ctx->u.br_en) { + BrotliEncoderSetParameter(ctx->u.br_en, + BROTLI_PARAM_MODE, BROTLI_MODE_TEXT); + BrotliEncoderSetParameter(ctx->u.br_en, + BROTLI_PARAM_QUALITY, BROTLI_MIN_QUALITY); + } + } + else + ctx->u.br_de = BrotliDecoderCreateInstance(NULL, NULL, NULL); + + return !ctx->u.br_de; +} + +static int +lcs_process_brotli(lws_comp_ctx_t *ctx, const void *in, size_t *ilen_iused, + void *out, size_t *olen_oused) +{ + size_t a_in, a_out, t_out; + const uint8_t *n_in; + uint8_t *n_out; + int n; + + n_in = (void *)in; + a_in = *ilen_iused; + a_out = *olen_oused; + n_out = out; + t_out = 0; + + if (!ctx->is_decompression) { + + if (!a_in && !BrotliEncoderHasMoreOutput(ctx->u.br_en)) { + *olen_oused = 0; + + goto bail; + } + + n = BROTLI_OPERATION_PROCESS; + if (!ctx->buflist_comp && ctx->final_on_input_side) + n = BROTLI_OPERATION_FINISH; + + if (BrotliEncoderCompressStream(ctx->u.br_en, n, &a_in, &n_in, + &a_out, &n_out, &t_out) == + BROTLI_FALSE) { + lwsl_err("brotli encode failed\n"); + + return -1; + } + + ctx->may_have_more = !a_out;//!BrotliEncoderIsFinished(ctx->u.br_en); + + } else { + n = BrotliDecoderDecompressStream(ctx->u.br_de, &a_in, &n_in, + &a_out, &n_out, &t_out); + + switch (n) { + case BROTLI_DECODER_RESULT_ERROR: + lwsl_err("brotli decoder error\n"); + return -1; + } + } + + *ilen_iused -= a_in; + *olen_oused -= a_out; + +bail: + if (!ctx->is_decompression) + return BrotliEncoderIsFinished(ctx->u.br_en); + else + return BrotliDecoderIsFinished(ctx->u.br_de); +} + +static void +lcs_destroy_brotli(lws_comp_ctx_t *ctx) +{ + if (!ctx) + return; + + if (!(*ctx).is_decompression) + BrotliEncoderDestroyInstance((*ctx).u.br_en); + else + BrotliDecoderDestroyInstance((*ctx).u.br_de); + + (*ctx).u.generic_ctx_ptr = NULL; +} + +struct lws_compression_support lcs_brotli = { + /* .encoding_name */ "br", + /* .init_compression */ lcs_init_compression_brotli, + /* .process */ lcs_process_brotli, + /* .destroy */ lcs_destroy_brotli, +}; diff --git a/lib/roles/http/compression/deflate/deflate.c b/lib/roles/http/compression/deflate/deflate.c new file mode 100644 index 000000000..2f3fab5e6 --- /dev/null +++ b/lib/roles/http/compression/deflate/deflate.c @@ -0,0 +1,110 @@ +/* + * libwebsockets - small server side websockets and web server implementation + * + * Copyright (C) 2010 - 2018 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include "core/private.h" + +static int +lcs_init_compression_deflate(lws_comp_ctx_t *ctx, int decomp) +{ + int n; + + ctx->is_decompression = decomp; + ctx->u.deflate = lws_malloc(sizeof(*ctx->u.deflate), __func__); + + if (!ctx->u.deflate) + return 2; + + memset(ctx->u.deflate, 0, sizeof(*ctx->u.deflate)); + + if (!decomp && + (n = deflateInit2(ctx->u.deflate, 1, Z_DEFLATED, -15, 8, + Z_DEFAULT_STRATEGY)) != Z_OK) { + lwsl_err("deflate init failed: %d\n", n); + lws_free_set_NULL(ctx->u.deflate); + + return 1; + } + + if (decomp && + inflateInit2(ctx->u.deflate, 16 + 15) != Z_OK) { + lws_free_set_NULL(ctx->u.deflate); + return 1; + } + + return 0; +} + +static int +lcs_process_deflate(lws_comp_ctx_t *ctx, const void *in, size_t *ilen_iused, + void *out, size_t *olen_oused) +{ + size_t olen_oused_in = *olen_oused; + int n; + + ctx->u.deflate->next_in = (void *)in; + ctx->u.deflate->avail_in = *ilen_iused; + + ctx->u.deflate->next_out = out; + ctx->u.deflate->avail_out = *olen_oused; + + if (!ctx->is_decompression) + n = deflate(ctx->u.deflate, Z_SYNC_FLUSH); + else + n = inflate(ctx->u.deflate, Z_SYNC_FLUSH); + + switch (n) { + case Z_NEED_DICT: + case Z_STREAM_ERROR: + case Z_DATA_ERROR: + case Z_MEM_ERROR: + lwsl_err("zlib error inflate %d\n", n); + return -1; + } + + *ilen_iused -= ctx->u.deflate->avail_in; + *olen_oused -= ctx->u.deflate->avail_out; + + /* it's ambiguous with zlib... */ + ctx->may_have_more = (*olen_oused == olen_oused_in); + + return n == Z_STREAM_END; +} + +static void +lcs_destroy_deflate(lws_comp_ctx_t *ctx) +{ + if (!ctx) + return; + + if (!(*ctx).is_decompression) + deflateEnd((*ctx).u.deflate); + else + inflateEnd((*ctx).u.deflate); + + lws_free_set_NULL(ctx->u.deflate); +} + +struct lws_compression_support lcs_deflate = { + /* .encoding_name */ "deflate", + /* .init_compression */ lcs_init_compression_deflate, + /* .process */ lcs_process_deflate, + /* .destroy */ lcs_destroy_deflate, +}; diff --git a/lib/roles/http/compression/private.h b/lib/roles/http/compression/private.h new file mode 100644 index 000000000..dafa77c77 --- /dev/null +++ b/lib/roles/http/compression/private.h @@ -0,0 +1,80 @@ +/* + * libwebsockets - small server side websockets and web server implementation + * + * Copyright (C) 2010 - 2018 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * This is included from core/private.h if LWS_WITH_HTTP_STREAM_COMPRESSION + */ + +#include +#if defined(LWS_WITH_HTTP_BROTLI) +#include +#include +#endif + +/* + * struct holding union of all the available compression methods' context data, + * and state if it's compressing or decompressing + */ + +typedef struct lws_compression_ctx { + union { + +#if defined(LWS_WITH_HTTP_BROTLI) + BrotliEncoderState *br_en; + BrotliDecoderState *br_de; +#endif + z_stream *deflate; + void *generic_ctx_ptr; + } u; + + struct lws_buflist *buflist_comp; + + unsigned int is_decompression:1; + unsigned int final_on_input_side:1; + unsigned int may_have_more:1; + unsigned int chunking:1; +} lws_comp_ctx_t; + +/* generic structure defining the interface to a compression method */ + +struct lws_compression_support { + /** compression name as used by, eg, content-ecoding */ + const char *encoding_name; + /** create a compression context for the compression method, or NULL */ + int (*init_compression)(lws_comp_ctx_t *ctx, int decomp); + /** pass data into the context to be processed */ + int (*process)(lws_comp_ctx_t *ctx, const void *in, size_t *ilen_iused, + void *out, size_t *olen_oused); + /** destroy the de/compression context */ + void (*destroy)(lws_comp_ctx_t *ctx); +}; + +extern struct lws_compression_support lcs_deflate; +extern struct lws_compression_support lcs_brotli; + +int +lws_http_compression_validate(struct lws *wsi); + +int +lws_http_compression_transform(struct lws *wsi, unsigned char *buf, + size_t len, enum lws_write_protocol *wp, + unsigned char **outbuf, size_t *olen_oused); + +void +lws_http_compression_destroy(struct lws *wsi); diff --git a/lib/roles/http/compression/stream.c b/lib/roles/http/compression/stream.c new file mode 100644 index 000000000..617750ad4 --- /dev/null +++ b/lib/roles/http/compression/stream.c @@ -0,0 +1,221 @@ +/* + * libwebsockets - small server side websockets and web server implementation + * + * Copyright (C) 2010 - 2018 Andy Green + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation: + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301 USA + */ + +#include "core/private.h" + +/* compression methods listed in order of preference */ + +struct lws_compression_support *lcs_available[] = { +#if defined(LWS_WITH_HTTP_BROTLI) + &lcs_brotli, +#endif + &lcs_deflate, +}; + +/* compute acceptable compression encodings while we still have an ah */ + +int +lws_http_compression_validate(struct lws *wsi) +{ + const char *a; + size_t n; + + wsi->http.comp_accept_mask = 0; + + if (!wsi->http.ah || !lwsi_role_server(wsi)) + return 0; + + a = lws_hdr_simple_ptr(wsi, WSI_TOKEN_HTTP_ACCEPT_ENCODING); + if (!a) + return 0; + + for (n = 0; n < LWS_ARRAY_SIZE(lcs_available); n++) + if (strstr(a, lcs_available[n]->encoding_name)) + wsi->http.comp_accept_mask |= 1 << n; + + return 0; +} + +LWS_VISIBLE int +lws_http_compression_apply(struct lws *wsi, const char *name, + unsigned char **p, unsigned char *end, char decomp) +{ + size_t n; + + for (n = 0; n < LWS_ARRAY_SIZE(lcs_available); n++) { + /* if name is non-NULL, choose only that compression method */ + if (name && !strcmp(lcs_available[n]->encoding_name, name)) + continue; + /* + * If we're the server, confirm that the client told us he could + * handle this kind of compression transform... + */ + if (!decomp && !(wsi->http.comp_accept_mask & (1 << n))) + continue; + + /* let's go with this one then... */ + break; + } + + if (n == LWS_ARRAY_SIZE(lcs_available)) + return 1; + + lcs_available[n]->init_compression(&wsi->http.comp_ctx, decomp); + if (!wsi->http.comp_ctx.u.generic_ctx_ptr) { + lwsl_err("%s: init_compression %d failed\n", __func__, (int)n); + return 1; + } + + wsi->http.lcs = lcs_available[n]; + wsi->http.comp_ctx.may_have_more = 0; + wsi->http.comp_ctx.final_on_input_side = 0; + wsi->http.comp_ctx.chunking = 0; + wsi->http.comp_ctx.is_decompression = decomp; + + if (lws_add_http_header_by_token(wsi, WSI_TOKEN_HTTP_CONTENT_ENCODING, + (unsigned char *)lcs_available[n]->encoding_name, + strlen(lcs_available[n]->encoding_name), p, end)) + return -1; + + lwsl_info("%s: wsi %p: applied %s content-encoding\n", __func__, + wsi, lcs_available[n]->encoding_name); + + return 0; +} + +void +lws_http_compression_destroy(struct lws *wsi) +{ + if (!wsi->http.lcs || !wsi->http.comp_ctx.u.generic_ctx_ptr) + return; + + wsi->http.lcs->destroy(&wsi->http.comp_ctx); + + wsi->http.lcs = NULL; +} + +/* + * This manages the compression transform independent of h1 or h2. + * + * wsi->buflist_comp stashes pre-transform input that was not yet compressed + */ + +int +lws_http_compression_transform(struct lws *wsi, unsigned char *buf, + size_t len, enum lws_write_protocol *wp, + unsigned char **outbuf, size_t *olen_oused) +{ + size_t ilen_iused = len; + int n, use = 0, wp1f = (*wp) & 0x1f; + lws_comp_ctx_t *ctx = &wsi->http.comp_ctx; + + ctx->may_have_more = 0; + + if (!wsi->http.lcs || + (wp1f != LWS_WRITE_HTTP && wp1f != LWS_WRITE_HTTP_FINAL)) { + *outbuf = buf; + *olen_oused = len; + + return 0; + } + + if (wp1f == LWS_WRITE_HTTP_FINAL) { + /* + * ...we may get a large buffer that represents the final input + * buffer, but it may form multiple frames after being + * tranformed by compression; only the last of those is actually + * the final frame on the output stream. + * + * Note that we have received the FINAL input, and downgrade it + * to a non-final for now. + */ + ctx->final_on_input_side = 1; + *wp = LWS_WRITE_HTTP | ((*wp) & ~0x1f); + } + + if (ctx->buflist_comp || ctx->may_have_more) { + /* + * we can't send this new stuff when we have old stuff + * buffered and not compressed yet. Add it to the tail + * and switch to trying to process the head. + */ + if (buf && len) { + lws_buflist_append_segment( + &ctx->buflist_comp, buf, len); + lwsl_debug("%s: %p: adding %d to comp buflist\n", + __func__,wsi, (int)len); + } + + len = lws_buflist_next_segment_len(&ctx->buflist_comp, &buf); + ilen_iused = len; + use = 1; + lwsl_debug("%s: %p: trying comp buflist %d\n", __func__, wsi, + (int)len); + } + + if (!buf && ilen_iused) + return 0; + + lwsl_debug("%s: %p: pre-process: ilen_iused %d, olen_oused %d\n", + __func__, wsi, (int)ilen_iused, (int)*olen_oused); + + n = wsi->http.lcs->process(ctx, buf, &ilen_iused, *outbuf, olen_oused); + + if (n && n != 1) { + lwsl_err("%s: problem with compression\n", __func__); + + return -1; + } + + if (!ctx->may_have_more && ctx->final_on_input_side) + *wp = LWS_WRITE_HTTP_FINAL | ((*wp) & ~0x1f); + + lwsl_debug("%s: %p: more %d, ilen_iused %d\n", __func__, wsi, + ctx->may_have_more, (int)ilen_iused); + + if (use && ilen_iused) { + /* + * we were flushing stuff from the buflist head... account for + * however much actually got processed by the compression + * transform + */ + lws_buflist_use_segment(&ctx->buflist_comp, ilen_iused); + lwsl_debug("%s: %p: marking %d of comp buflist as used " + "(ctx->buflist_comp %p)\n", __func__, wsi, + (int)len, ctx->buflist_comp); + } + + if (!use && ilen_iused != len) { + /* + * ...we were sending stuff from the caller directly and not + * all of it got processed... stash on the buflist tail + */ + lws_buflist_append_segment(&ctx->buflist_comp, + buf + ilen_iused, len - ilen_iused); + + lwsl_debug("%s: buffering %d unused comp input\n", __func__, + (int)(len - ilen_iused)); + } + if (ctx->buflist_comp || ctx->may_have_more) + lws_callback_on_writable(wsi); + + return 0; +} diff --git a/lib/roles/http/header.c b/lib/roles/http/header.c index ca2e6dad3..d99c3c836 100644 --- a/lib/roles/http/header.c +++ b/lib/roles/http/header.c @@ -413,3 +413,20 @@ lws_http_redirect(struct lws *wsi, int code, const unsigned char *loc, int len, return lws_write(wsi, start, *p - start, LWS_WRITE_HTTP_HEADERS | LWS_WRITE_H2_STREAM_END); } + +#if !defined(LWS_WITH_HTTP_STREAM_COMPRESSION) +LWS_VISIBLE int +lws_http_compression_apply(struct lws *wsi, const char *name, + unsigned char **p, unsigned char *end, char decomp) +{ + (void)wsi; + (void)name; + (void)p; + (void)end; + (void)decomp; + + return 0; +} +#endif + + diff --git a/lib/roles/http/private.h b/lib/roles/http/private.h index d54f4e44b..f901ea08e 100644 --- a/lib/roles/http/private.h +++ b/lib/roles/http/private.h @@ -27,6 +27,10 @@ #include #endif +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) +#include "roles/http/compression/private.h" +#endif + #define lwsi_role_http(wsi) (lwsi_role_h1(wsi) || lwsi_role_h2(wsi)) enum http_version { @@ -192,6 +196,9 @@ struct lws_access_log { }; #endif +#define LWS_HTTP_CHUNK_HDR_MAX_SIZE (6 + 2) /* 6 hex digits and then CRLF */ +#define LWS_HTTP_CHUNK_TRL_MAX_SIZE (2 + 5) /* CRLF, then maybe 0 CRLF CRLF */ + struct _lws_http_mode_related { struct lws *new_wsi_list; @@ -216,6 +223,10 @@ struct _lws_http_mode_related { #ifdef LWS_WITH_CGI struct lws_cgi *cgi; /* wsi being cgi master have one of these */ #endif +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + struct lws_compression_support *lcs; + lws_comp_ctx_t comp_ctx; +#endif enum http_version request_version; enum http_connection_type connection_type; diff --git a/lib/roles/http/server/parsers.c b/lib/roles/http/server/parsers.c index 1c0f6564e..ca05a7836 100644 --- a/lib/roles/http/server/parsers.c +++ b/lib/roles/http/server/parsers.c @@ -529,6 +529,9 @@ char *lws_hdr_simple_ptr(struct lws *wsi, enum lws_token_indexes h) { int n; + if (!wsi->http.ah) + return NULL; + n = wsi->http.ah->frag_index[h]; if (!n) return NULL; @@ -539,6 +542,9 @@ char *lws_hdr_simple_ptr(struct lws *wsi, enum lws_token_indexes h) static int LWS_WARN_UNUSED_RESULT lws_pos_in_bounds(struct lws *wsi) { + if (!wsi->http.ah) + return -1; + if (wsi->http.ah->pos < (unsigned int)wsi->context->max_http_header_data) return 0; diff --git a/lib/roles/http/server/server.c b/lib/roles/http/server/server.c index 666a16c74..9930632df 100644 --- a/lib/roles/http/server/server.c +++ b/lib/roles/http/server/server.c @@ -1713,7 +1713,12 @@ lws_http_transaction_completed(struct lws *wsi) { int n = NO_PENDING_TIMEOUT; - if (lws_has_buffered_out(wsi)) { + if (lws_has_buffered_out(wsi) +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + || wsi->http.comp_ctx.buflist_comp || + wsi->http.comp_ctx.may_have_more +#endif + ) { /* * ...so he tried to send something large as the http reply, * it went as a partial, but he immediately said the @@ -1722,14 +1727,18 @@ lws_http_transaction_completed(struct lws *wsi) * Defer the transaction completed until the last part of the * partial is sent. */ - lwsl_notice("%s: deferring due to partial\n", __func__); + lwsl_debug("%s: %p: deferring due to partial\n", __func__, wsi); wsi->http.deferred_transaction_completed = 1; + lws_callback_on_writable(wsi); return 0; } lwsl_info("%s: wsi %p\n", __func__, wsi); +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + lws_http_compression_destroy(wsi); +#endif lws_access_log(wsi); if (!wsi->hdr_parsing_completed) { @@ -1764,6 +1773,7 @@ lws_http_transaction_completed(struct lws *wsi) wsi->http.tx_content_length = 0; wsi->http.tx_content_remain = 0; wsi->hdr_parsing_completed = 0; + wsi->sending_chunked = 0; #ifdef LWS_WITH_ACCESS_LOG wsi->http.access_log.sent = 0; #endif @@ -1922,6 +1932,20 @@ lws_serve_http_file(struct lws *wsi, const char *file, const char *content_type, return -1; lwsl_info("file is being provided in gzip\n"); } +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + else { + /* + * if we know its very compressible, and we can use + * compression, then use the most preferred compression + * method that the client said he will accept + */ + + if (!strncmp(content_type, "text/", 5) || + !strcmp(content_type, "application/javascript") || + !strcmp(content_type, "image/svg+xml")) + lws_http_compression_apply(wsi, NULL, &p, end, 0); + } +#endif if ( #if defined(LWS_WITH_RANGES) @@ -2003,17 +2027,46 @@ lws_serve_http_file(struct lws *wsi, const char *file, const char *content_type, #endif if (!wsi->http2_substream) { - if (!wsi->sending_chunked) { + /* for http/1.1 ... */ + if (!wsi->sending_chunked +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + && !wsi->http.lcs +#endif + ) { + /* ... if not already using chunked and not using an + * http compression translation, then send the naive + * content length + */ if (lws_add_http_header_content_length(wsi, - total_content_length, - &p, end)) + total_content_length, &p, end)) return -1; } else { + /* ...otherwise, for http 1 it must go chunked. For + * the compression case, the reason is we compress on + * the fly and do not know the compressed content-length + * until it has all been sent. Http/1.1 pipelining must + * be able to know where the transaction boundaries are + * ... so chunking... + */ if (lws_add_http_header_by_token(wsi, - WSI_TOKEN_HTTP_TRANSFER_ENCODING, - (unsigned char *)"chunked", - 7, &p, end)) + WSI_TOKEN_HTTP_TRANSFER_ENCODING, + (unsigned char *)"chunked", 7, &p, end)) return -1; + +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + if (wsi->http.lcs) { + /* + * ...this is fun, isn't it :-) For h1 that is + * using an http compression translation, the + * compressor must chunk its output privately. + * + * h2 doesn't need (or support) any of this + * crap. + */ + lwsl_debug("setting chunking\n"); + wsi->http.comp_ctx.chunking = 1; + } +#endif } } @@ -2082,6 +2135,8 @@ LWS_VISIBLE int lws_serve_http_file_fragment(struct lws *wsi) do { + /* priority 1: buffered output */ + if (lws_has_buffered_out(wsi)) { if (lws_issue_raw(wsi, NULL, 0) < 0) { lwsl_info("%s: closing\n", __func__); @@ -2090,6 +2145,27 @@ LWS_VISIBLE int lws_serve_http_file_fragment(struct lws *wsi) break; } + /* priority 2: buffered pre-compression-transform */ + +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + if (wsi->http.comp_ctx.buflist_comp || + wsi->http.comp_ctx.may_have_more) { + enum lws_write_protocol wp = LWS_WRITE_HTTP; + + lwsl_debug("%s: completing comp partial (buflist_comp %p, may %d)\n", + __func__, wsi->http.comp_ctx.buflist_comp, + wsi->http.comp_ctx.may_have_more); + + if (wsi->role_ops->write_role_protocol(wsi, NULL, 0, &wp) < 0) { + lwsl_info("%s signalling to close\n", __func__); + goto file_had_it; + } + lws_callback_on_writable(wsi); + + break; + } +#endif + if (wsi->http.filepos == wsi->http.filelen) goto all_sent; @@ -2257,12 +2333,18 @@ LWS_VISIBLE int lws_serve_http_file_fragment(struct lws *wsi) } all_sent: - if ((!lws_has_buffered_out(wsi) && wsi->http.filepos >= wsi->http.filelen) + if ((!lws_has_buffered_out(wsi) +#if defined(LWS_WITH_HTTP_STREAM_COMPRESSION) + && !wsi->http.comp_ctx.buflist_comp && + !wsi->http.comp_ctx.may_have_more +#endif + ) && (wsi->http.filepos >= wsi->http.filelen #if defined(LWS_WITH_RANGES) || finished) #else ) #endif + ) { lwsi_set_state(wsi, LRS_ESTABLISHED); /* we might be in keepalive, so close it off here */ @@ -2297,7 +2379,7 @@ all_sent: return 1; /* >0 indicates completed */ } - } while (0); // while (!lws_send_pipe_choked(wsi)) + } while (1); //(!lws_send_pipe_choked(wsi)); lws_callback_on_writable(wsi); diff --git a/lib/tls/mbedtls/ssl.c b/lib/tls/mbedtls/ssl.c index f311ef50e..da25a2712 100644 --- a/lib/tls/mbedtls/ssl.c +++ b/lib/tls/mbedtls/ssl.c @@ -189,7 +189,7 @@ lws_ssl_capable_write(struct lws *wsi, unsigned char *buf, int len) if (m == SSL_ERROR_WANT_WRITE || SSL_want_write(wsi->tls.ssl)) { lws_set_blocking_send(wsi); - lwsl_notice("%s: want write\n", __func__); + lwsl_debug("%s: want write\n", __func__); return LWS_SSL_CAPABLE_MORE_SERVICE; }