From 6710279e21effbc2b1a033bbce0ee911f2555074 Mon Sep 17 00:00:00 2001 From: Andy Green Date: Tue, 27 Aug 2019 06:06:13 +0100 Subject: [PATCH] client: use block parse and buflist With http, the protocol doesn't indicate where the headers end and the next transaction or body begin. Until now, we handled that for client header response parsing by reading from the tls buffer bytewise. This modernizes the code to read in up to 256-byte chunks and parse the chunks in one hit (the parse API is already set up for doing this elsewhere). Now we have a generic input buflist, adapt the parser loop to go through that and arrange that any leftovers are placed on there. --- lib/core-net/private-lib-core-net.h | 4 +- lib/core-net/service.c | 12 +-- lib/core/buflist.c | 2 +- lib/core/context.c | 2 +- lib/roles/h1/ops-h1.c | 6 +- lib/roles/h2/ops-h2.c | 2 + lib/roles/http/client/client-http.c | 111 +++++++++++++++++++--------- lib/roles/http/header.c | 4 +- lib/roles/http/server/server.c | 3 +- lib/roles/raw-proxy/ops-raw-proxy.c | 4 +- lib/roles/raw-skt/ops-raw-skt.c | 4 +- lib/roles/ws/ops-ws.c | 3 +- 12 files changed, 105 insertions(+), 52 deletions(-) diff --git a/lib/core-net/private-lib-core-net.h b/lib/core-net/private-lib-core-net.h index 2fd63d631..afafbc11b 100644 --- a/lib/core-net/private-lib-core-net.h +++ b/lib/core-net/private-lib-core-net.h @@ -1193,8 +1193,8 @@ int lws_buflist_aware_read(struct lws_context_per_thread *pt, struct lws *wsi, struct lws_tokens *ebuf, const char *hint); int -lws_buflist_aware_consume(struct lws *wsi, struct lws_tokens *ebuf, int used, - int buffered, const char *hint); +lws_buflist_aware_finished_consuming(struct lws *wsi, struct lws_tokens *ebuf, + int used, int buffered, const char *hint); extern const struct lws_protocols protocol_abs_client_raw_skt, protocol_abs_client_unit_test; diff --git a/lib/core-net/service.c b/lib/core-net/service.c index c83af8e80..18af9c6e2 100644 --- a/lib/core-net/service.c +++ b/lib/core-net/service.c @@ -356,9 +356,9 @@ lws_buflist_aware_read(struct lws_context_per_thread *pt, struct lws *wsi, // lws_buflist_describe(&wsi->buflist, wsi, __func__); (void)hint; - ebuf->token = pt->serv_buf; - n = lws_ssl_capable_read(wsi, pt->serv_buf, - wsi->context->pt_serv_buf_size); + ebuf->token = pt->serv_buf + LWS_PRE; + n = lws_ssl_capable_read(wsi, pt->serv_buf + LWS_PRE, + wsi->context->pt_serv_buf_size - LWS_PRE); ebuf->len = n; lwsl_info("%s: wsi %p: %s: ssl_capable_read %d (prior %d)\n", __func__, @@ -405,8 +405,8 @@ get_from_buflist: } int -lws_buflist_aware_consume(struct lws *wsi, struct lws_tokens *ebuf, int used, - int buffered, const char *hint) +lws_buflist_aware_finished_consuming(struct lws *wsi, struct lws_tokens *ebuf, + int used, int buffered, const char *hint) { struct lws_context_per_thread *pt = &wsi->context->pt[(int)wsi->tsi]; int m; @@ -437,6 +437,8 @@ lws_buflist_aware_consume(struct lws *wsi, struct lws_tokens *ebuf, int used, /* any remainder goes on the buflist */ if (used != ebuf->len) { + // lwsl_notice("%s %s bac appending %d\n", __func__, hint, + // ebuf->len - used); m = lws_buflist_append_segment(&wsi->buflist, ebuf->token + used, ebuf->len - used); diff --git a/lib/core/buflist.c b/lib/core/buflist.c index 595cb34dd..bb86b4cc5 100644 --- a/lib/core/buflist.c +++ b/lib/core/buflist.c @@ -166,7 +166,7 @@ lws_buflist_describe(struct lws_buflist **head, void *id, const char *reason) int n = 0; if (*head == NULL) - lwsl_notice("%p: %sL buflist empty\n", id, reason); + lwsl_notice("%p: %s: buflist empty\n", id, reason); while (*head) { lwsl_notice("%p: %s: %d: %llu / %llu (%llu left)\n", id, diff --git a/lib/core/context.c b/lib/core/context.c index ca32249d1..0372c88be 100644 --- a/lib/core/context.c +++ b/lib/core/context.c @@ -102,7 +102,6 @@ lws_create_context(const struct lws_context_creation_info *info) #endif lwsl_info(" LWS_DEF_HEADER_LEN : %u\n", LWS_DEF_HEADER_LEN); - lwsl_info(" LWS_MAX_PROTOCOLS : %u\n", LWS_MAX_PROTOCOLS); lwsl_info(" LWS_MAX_SMP : %u\n", LWS_MAX_SMP); lwsl_info(" sizeof (*info) : %ld\n", (long)sizeof(*info)); #if defined(LWS_WITH_STATS) @@ -127,6 +126,7 @@ lws_create_context(const struct lws_context_creation_info *info) if (info->pt_serv_buf_size) s1 = info->pt_serv_buf_size; + /* pt fakewsi and the pt serv buf allocations ride after the context */ size += count_threads * (s1 + sizeof(struct lws)); #endif diff --git a/lib/roles/h1/ops-h1.c b/lib/roles/h1/ops-h1.c index 4788c84aa..4a6fbf08f 100644 --- a/lib/roles/h1/ops-h1.c +++ b/lib/roles/h1/ops-h1.c @@ -389,7 +389,8 @@ lws_h1_server_socket_service(struct lws *wsi, struct lws_pollfd *pollfd) if (lwsi_state(wsi) == LRS_ISSUING_FILE) { // lwsl_notice("stashing: wsi %p: bd %d\n", wsi, buffered); - if (lws_buflist_aware_consume(wsi, &ebuf, 0, buffered, __func__)) + if (lws_buflist_aware_finished_consuming(wsi, &ebuf, 0, + buffered, __func__)) return LWS_HPI_RET_PLEASE_CLOSE_ME; goto try_pollout; @@ -410,7 +411,8 @@ lws_h1_server_socket_service(struct lws *wsi, struct lws_pollfd *pollfd) // lwsl_notice("%s: consumed %d\n", __func__, n); - if (lws_buflist_aware_consume(wsi, &ebuf, n, buffered, __func__)) + if (lws_buflist_aware_finished_consuming(wsi, &ebuf, n, + buffered, __func__)) return LWS_HPI_RET_PLEASE_CLOSE_ME; /* diff --git a/lib/roles/h2/ops-h2.c b/lib/roles/h2/ops-h2.c index 6f4178ff2..dd57ce4f7 100644 --- a/lib/roles/h2/ops-h2.c +++ b/lib/roles/h2/ops-h2.c @@ -273,6 +273,7 @@ drain: } if (n && buffered) { + // lwsl_notice("%s: h2 use %d\n", __func__, n); m = lws_buflist_use_segment(&wsi->buflist, n); lwsl_info("%s: draining rxflow: used %d, next %d\n", __func__, n, m); @@ -283,6 +284,7 @@ drain: } } else if (n && n != ebuf.len) { + // lwsl_notice("%s: h2 append seg %d\n", __func__, ebuf.len - n); m = lws_buflist_append_segment(&wsi->buflist, ebuf.token + n, ebuf.len - n); diff --git a/lib/roles/http/client/client-http.c b/lib/roles/http/client/client-http.c index 5c0fe0296..d23474731 100644 --- a/lib/roles/http/client/client-http.c +++ b/lib/roles/http/client/client-http.c @@ -76,10 +76,6 @@ lws_client_socket_service(struct lws *wsi, struct lws_pollfd *pollfd, char ebuf[128]; #endif const char *cce = NULL; -#if defined(LWS_ROLE_H1) || defined(LWS_ROLE_H2) - ssize_t len = 0; - unsigned char c; -#endif char *sb = p; int n = 0; #if defined(LWS_WITH_SOCKS5) @@ -544,26 +540,45 @@ client_http_body_sent: * in one packet, since at that point the connection is * definitively ready from browser pov. */ - len = 1; - while (wsi->http.ah->parser_state != WSI_PARSING_COMPLETE && - len > 0) { - int plen = 1; + while (wsi->http.ah->parser_state != WSI_PARSING_COMPLETE) { + struct lws_tokens eb; + int n, m, buffered; - n = lws_ssl_capable_read(wsi, &c, 1); - switch (n) { - case 0: - case LWS_SSL_CAPABLE_ERROR: + eb.token = NULL; + eb.len = 0; + buffered = lws_buflist_aware_read(pt, wsi, &eb, __func__); + lwsl_debug("%s: buflist-aware-read %d %d\n", __func__, + buffered, eb.len); + if (eb.len == LWS_SSL_CAPABLE_MORE_SERVICE) + return 0; + if (buffered < 0 || eb.len < 0) { cce = "read failed"; goto bail3; - case LWS_SSL_CAPABLE_MORE_SERVICE: - return 0; } + if (!eb.len) + return 0; - if (lws_parse(wsi, &c, &plen)) { + n = eb.len; + if (lws_parse(wsi, eb.token, &n)) { lwsl_warn("problems parsing header\n"); cce = "problems parsing header"; goto bail3; } + + m = eb.len - n; + if (lws_buflist_aware_finished_consuming(wsi, &eb, m, + buffered, + __func__)) + return -1; + eb.token += m; + eb.len -= m; + + if (n) { + assert(wsi->http.ah->parser_state == + WSI_PARSING_COMPLETE); + + break; + } } /* @@ -573,7 +588,6 @@ client_http_body_sent: */ if (wsi->http.ah->parser_state != WSI_PARSING_COMPLETE) break; - #endif /* @@ -1198,26 +1212,38 @@ lws_generate_client_handshake(struct lws *wsi, char *pkt) LWS_VISIBLE int lws_http_client_read(struct lws *wsi, char **buf, int *len) { - int rlen, n; + struct lws_context_per_thread *pt = &wsi->context->pt[(int)wsi->tsi]; + struct lws_tokens eb; + int buffered, n, consumed = 0; - rlen = lws_ssl_capable_read(wsi, (unsigned char *)*buf, *len); + eb.token = NULL; + eb.len = 0; + + buffered = lws_buflist_aware_read(pt, wsi, &eb, __func__); + *buf = (char *)eb.token; *len = 0; - // lwsl_notice("%s: rlen %d\n", __func__, rlen); + /* + * we're taking on responsibility for handling used / unused eb + * when we leave, via lws_buflist_aware_finished_consuming() + */ + +// lwsl_notice("%s: eb.len %d ENTRY chunk remaining %d\n", __func__, eb.len, +// wsi->chunk_remaining); /* allow the source to signal he has data again next time */ if (lws_change_pollfd(wsi, 0, LWS_POLLIN)) return -1; - if (rlen == LWS_SSL_CAPABLE_ERROR) { + if (buffered < 0) { lwsl_debug("%s: SSL capable error\n", __func__); return -1; } - if (rlen <= 0) + if (eb.len <= 0) return 0; - *len = rlen; + *len = eb.len; wsi->client_rx_avail = 0; /* @@ -1225,6 +1251,8 @@ lws_http_client_read(struct lws *wsi, char **buf, int *len) * so http client must deal with it */ spin_chunks: + //lwsl_notice("%s: len %d SPIN chunk remaining %d\n", __func__, *len, + // wsi->chunk_remaining); while (wsi->chunked && (wsi->chunk_parser != ELCP_CONTENT) && *len) { switch (wsi->chunk_parser) { case ELCP_HEX: @@ -1234,7 +1262,7 @@ spin_chunks: } n = char_to_hex((*buf)[0]); if (n < 0) { - lwsl_info("%s: chunking failure\n", __func__); + lwsl_err("%s: chunking failure A\n", __func__); return -1; } wsi->chunk_remaining <<= 4; @@ -1242,11 +1270,12 @@ spin_chunks: break; case ELCP_CR: if ((*buf)[0] != '\x0a') { - lwsl_info("%s: chunking failure\n", __func__); + lwsl_err("%s: chunking failure B\n", __func__); return -1; } wsi->chunk_parser = ELCP_CONTENT; - lwsl_info("chunk %d\n", wsi->chunk_remaining); + //lwsl_info("starting chunk size %d (block rem %d)\n", + // wsi->chunk_remaining, *len); if (wsi->chunk_remaining) break; lwsl_info("final chunk\n"); @@ -1257,7 +1286,8 @@ spin_chunks: case ELCP_POST_CR: if ((*buf)[0] != '\x0d') { - lwsl_info("%s: chunking failure\n", __func__); + lwsl_err("%s: chunking failure C\n", __func__); + lwsl_hexdump_err(*buf, *len); return -1; } @@ -1267,7 +1297,7 @@ spin_chunks: case ELCP_POST_LF: if ((*buf)[0] != '\x0a') { - lwsl_info("%s: chunking failure\n", __func__); + lwsl_err("%s: chunking failure D\n", __func__); return -1; } @@ -1278,10 +1308,11 @@ spin_chunks: } (*buf)++; (*len)--; + consumed++; } if (wsi->chunked && !wsi->chunk_remaining) - return 0; + goto account_and_ret; if (wsi->http.rx_content_remain && wsi->http.rx_content_remain < (unsigned int)*len) @@ -1319,11 +1350,17 @@ spin_chunks: } } - if (wsi->chunked && wsi->chunk_remaining) { - (*buf) += n; + (*buf) += n; + *len -= n; + if (wsi->chunked && wsi->chunk_remaining) wsi->chunk_remaining -= n; - *len -= n; - } + + //lwsl_notice("chunk_remaining <- %d, block remaining %d\n", + // wsi->chunk_remaining, *len); + + consumed += n; + //eb.token += n; + //eb.len -= n; if (wsi->chunked && !wsi->chunk_remaining) wsi->chunk_parser = ELCP_POST_CR; @@ -1332,7 +1369,7 @@ spin_chunks: goto spin_chunks; if (wsi->chunked) - return 0; + goto account_and_ret; /* if we know the content length, decrement the content remaining */ if (wsi->http.rx_content_length > 0) @@ -1342,7 +1379,7 @@ spin_chunks: // wsi->http.rx_content_remain, wsi->http.rx_content_length); if (wsi->http.rx_content_remain || !wsi->http.rx_content_length) - return 0; + goto account_and_ret; completed: @@ -1351,6 +1388,12 @@ completed: return -1; } +account_and_ret: +// lwsl_warn("%s: on way out, consuming %d / %d\n", __func__, consumed, eb.len); + if (lws_buflist_aware_finished_consuming(wsi, &eb, consumed, buffered, + __func__)) + return -1; + return 0; } diff --git a/lib/roles/http/header.c b/lib/roles/http/header.c index 007bb00ef..167de36b1 100644 --- a/lib/roles/http/header.c +++ b/lib/roles/http/header.c @@ -352,11 +352,13 @@ lws_add_http_header_status(struct lws *wsi, unsigned int _code, } if (wsi->context->server_string && - !(_code & LWSAHH_FLAG_NO_SERVER_NAME)) + !(_code & LWSAHH_FLAG_NO_SERVER_NAME)) { + assert(wsi->context->server_string_len > 0); if (lws_add_http_header_by_token(wsi, WSI_TOKEN_HTTP_SERVER, (unsigned char *)wsi->context->server_string, wsi->context->server_string_len, p, end)) return 1; + } if (wsi->vhost->options & LWS_SERVER_OPTION_STS) if (lws_add_http_header_by_name(wsi, (unsigned char *) diff --git a/lib/roles/http/server/server.c b/lib/roles/http/server/server.c index 4c7a2a6fe..26168cae9 100644 --- a/lib/roles/http/server/server.c +++ b/lib/roles/http/server/server.c @@ -1700,7 +1700,8 @@ deal_body: if (m < 0) return -1; - if (lws_buflist_aware_consume(wsi, &ebuf, m, 1, __func__)) + if (lws_buflist_aware_finished_consuming(wsi, + &ebuf, m, 1, __func__)) return -1; } } diff --git a/lib/roles/raw-proxy/ops-raw-proxy.c b/lib/roles/raw-proxy/ops-raw-proxy.c index 8a5d8958c..d5368778f 100644 --- a/lib/roles/raw-proxy/ops-raw-proxy.c +++ b/lib/roles/raw-proxy/ops-raw-proxy.c @@ -86,8 +86,8 @@ rops_handle_POLLIN_raw_proxy(struct lws_context_per_thread *pt, struct lws *wsi, goto fail; } - if (lws_buflist_aware_consume(wsi, &ebuf, ebuf.len, buffered, - __func__)) + if (lws_buflist_aware_finished_consuming(wsi, &ebuf, ebuf.len, + buffered, __func__)) return LWS_HPI_RET_PLEASE_CLOSE_ME; } else if (wsi->favoured_pollin && diff --git a/lib/roles/raw-skt/ops-raw-skt.c b/lib/roles/raw-skt/ops-raw-skt.c index 793098a3d..d34ae94d7 100644 --- a/lib/roles/raw-skt/ops-raw-skt.c +++ b/lib/roles/raw-skt/ops-raw-skt.c @@ -101,8 +101,8 @@ rops_handle_POLLIN_raw_skt(struct lws_context_per_thread *pt, struct lws *wsi, goto fail; } - if (lws_buflist_aware_consume(wsi, &ebuf, ebuf.len, buffered, - __func__)) + if (lws_buflist_aware_finished_consuming(wsi, &ebuf, ebuf.len, + buffered, __func__)) return LWS_HPI_RET_PLEASE_CLOSE_ME; } else if (wsi->favoured_pollin && diff --git a/lib/roles/ws/ops-ws.c b/lib/roles/ws/ops-ws.c index 08e5ad1b4..0ca5a723c 100644 --- a/lib/roles/ws/ops-ws.c +++ b/lib/roles/ws/ops-ws.c @@ -1255,7 +1255,8 @@ drain: } //lws_buflist_describe(&wsi->buflist, wsi, __func__); //lwsl_notice("%s: consuming %d / %d\n", __func__, n, ebuf.len); - if (lws_buflist_aware_consume(wsi, &ebuf, n, buffered, __func__)) + if (lws_buflist_aware_finished_consuming(wsi, &ebuf, n, + buffered, __func__)) return LWS_HPI_RET_PLEASE_CLOSE_ME; }