http: added support for chunked transfer encoding (#90)

include/re_http.h

@@ -82,7 +82,8 @@ struct http_msg {
 	struct pl reason;      /**< Response Reason phrase                 */
 	struct list hdrl;      /**< List of HTTP headers (struct http_hdr) */
 	struct msg_ctype ctyp; /**< Content-type                           */
-	struct mbuf *mb;       /**< Buffer containing the HTTP message     */
+	struct mbuf *_mb;      /**< Buffer containing the HTTP message     */
+	struct mbuf *mb;       /**< Buffer containing the HTTP body        */
 	uint32_t clen;         /**< Content length                         */
 };
 
@@ -114,16 +115,20 @@ int  http_msg_print(struct re_printf *pf, const struct http_msg *msg);
 struct http_cli;
 struct http_req;
 struct dnsc;
+struct tcp_conn;
+struct tls_conn;
 
 typedef void (http_resp_h)(int err, const struct http_msg *msg, void *arg);
-typedef void (http_data_h)(struct mbuf *mb, void *arg);
+typedef int  (http_data_h)(const uint8_t *buf, size_t size,
+			   const struct http_msg *msg, void *arg);
+typedef void (http_conn_h)(struct tcp_conn *tc, struct tls_conn *sc,
+			   void *arg);
 
 int http_client_alloc(struct http_cli **clip, struct dnsc *dnsc);
 int http_request(struct http_req **reqp, struct http_cli *cli, const char *met,
 		 const char *uri, http_resp_h *resph, http_data_h *datah,
 		 void *arg, const char *fmt, ...);
-struct tcp_conn *http_req_tcp(struct http_req *req);
+void http_req_set_conn_handler(struct http_req *req, http_conn_h *connh);
-struct tls_conn *http_req_tls(struct http_req *req);
 
 
 /* Server */

src/http/chunk.c

@@ -0,0 +1,111 @@
+/**
+ * @file http/chunk.c Chunked Transfer Encoding
+ *
+ * Copyright (C) 2011 Creytiv.com
+ */
+
+#include <re_types.h>
+#include <re_mem.h>
+#include <re_mbuf.h>
+#include "http.h"
+
+
+static int decode_chunk_size(struct http_chunk *chunk, struct mbuf *mb)
+{
+	while (mbuf_get_left(mb)) {
+
+		char ch = (char)mbuf_read_u8(mb);
+		uint8_t c;
+
+		if (ch == '\n') {
+			if (chunk->digit) {
+				chunk->digit = false;
+				chunk->param = false;
+
+				return 0;
+			}
+			else
+				continue;
+		}
+
+		if (chunk->param)
+			continue;
+
+		if ('0' <= ch && ch <= '9')
+			c = ch - '0';
+		else if ('A' <= ch && ch <= 'F')
+			c = ch - 'A' + 10;
+		else if ('a' <= ch && ch <= 'f')
+			c = ch - 'a' + 10;
+		else if (ch == '\r' || ch == ' ' || ch == '\t')
+			continue;
+		else if (ch == ';' && chunk->digit) {
+			chunk->param = true;
+			continue;
+		}
+		else
+			return EPROTO;
+
+		chunk->digit = true;
+
+		chunk->size <<= 4;
+		chunk->size += c;
+	}
+
+	return ENODATA;
+}
+
+
+static int decode_trailer(struct http_chunk *chunk, struct mbuf *mb)
+{
+	while (mbuf_get_left(mb)) {
+
+		char ch = (char)mbuf_read_u8(mb);
+
+		if (ch == '\n') {
+			if (++chunk->lf >= 2)
+				return 0;
+		}
+		else if (ch != '\r')
+			chunk->lf = 0;
+	}
+
+	return ENODATA;
+}
+
+
+int http_chunk_decode(struct http_chunk *chunk, struct mbuf *mb, size_t *size)
+{
+	int err;
+
+	if (!chunk || !mb || !size)
+		return EINVAL;
+
+	if (chunk->trailer) {
+		err = decode_trailer(chunk, mb);
+		if (err)
+			return err;
+
+		*size = 0;
+
+		return 0;
+	}
+
+	err = decode_chunk_size(chunk, mb);
+	if (err)
+		return err;
+
+	if (chunk->size == 0) {
+		chunk->trailer = true;
+		chunk->lf = 1;
+
+		err = decode_trailer(chunk, mb);
+		if (err)
+			return err;
+	}
+
+	*size = chunk->size;
+	chunk->size = 0;
+
+	return 0;
+}

src/http/client.c

@@ -19,6 +19,7 @@
 #include <re_dns.h>
 #include <re_msg.h>
 #include <re_http.h>
+#include "http.h"
 
 
 enum {
@@ -39,10 +40,12 @@ struct http_cli {
 struct conn;
 
 struct http_req {
+	struct http_chunk chunk;
 	struct sa srvv[16];
 	struct le le;
 	struct http_req **reqp;
 	struct http_cli *cli;
+	struct http_msg *msg;
 	struct dns_query *dq;
 	struct conn *conn;
 	struct mbuf *mbreq;
@@ -50,14 +53,14 @@ struct http_req {
 	char *host;
 	http_resp_h *resph;
 	http_data_h *datah;
+	http_conn_h *connh;
 	void *arg;
-	size_t rx_bytes;
 	size_t rx_len;
 	unsigned srvc;
 	uint16_t port;
+	bool chunked;
 	bool secure;
 	bool close;
-	bool data;
 };
 
 
@@ -102,6 +105,7 @@ static void req_destructor(void *arg)
 	struct http_req *req = arg;
 
 	list_unlink(&req->le);
+	mem_deref(req->msg);
 	mem_deref(req->dq);
 	mem_deref(req->conn);
 	mem_deref(req->mbreq);
@@ -136,7 +140,10 @@ static void req_close(struct http_req *req, int err,
 	req->datah = NULL;
 
 	if (req->conn) {
-		if (err || req->close)
+		if (req->connh)
+			req->connh(req->conn->tc, req->conn->sc, req->arg);
+
+		if (err || req->close || req->connh)
 			mem_deref(req->conn);
 		else
 			conn_idle(req->conn);
@@ -144,12 +151,17 @@ static void req_close(struct http_req *req, int err,
 		req->conn = NULL;
 	}
 
+	req->connh = NULL;
+
 	if (req->reqp) {
 		*req->reqp = NULL;
 		req->reqp = NULL;
 	}
 
 	if (req->resph) {
+		if (msg)
+			msg->mb->pos = 0;
+
 		req->resph(err, msg, req->arg);
 		req->resph = NULL;
 	}
@@ -173,7 +185,7 @@ static void try_next(struct conn *conn, int err)
 	if (retry)
 		++req->srvc;
 
-	if (req->srvc > 0 && !req->data) {
+	if (req->srvc > 0 && !req->msg) {
 
 		err = req_connect(req);
 		if (!err)
@@ -184,27 +196,77 @@ static void try_next(struct conn *conn, int err)
 }
 
 
-static void req_recv(struct http_req *req, struct mbuf *mb)
+static int write_body_buf(struct http_msg *msg, const uint8_t *buf, size_t sz)
 {
-	uint32_t nrefs;
+	if ((msg->mb->pos + sz) > BUFSIZE_MAX)
+		return EOVERFLOW;
 
-	req->rx_bytes += mbuf_get_left(mb);
+	return mbuf_write_mem(msg->mb, buf, sz);
+}
 
-	mem_ref(req);
+
+static int write_body(struct http_req *req, struct mbuf *mb)
+{
+	const size_t size = min(mbuf_get_left(mb), req->rx_len);
+	int err;
+
+	if (size == 0)
+		return 0;
 
 	if (req->datah)
-		req->datah(mb, req->arg);
+		err = req->datah(mbuf_buf(mb), size, req->msg, req->arg);
+	else
+		err = write_body_buf(req->msg, mbuf_buf(mb), size);
 
-	nrefs = mem_nrefs(req);
+	if (err)
-	mem_deref(req);
+		return err;
 
-	if (nrefs == 1)
+	req->rx_len -= size;
-		return;
+	mb->pos     += size;
 
-	if (req->rx_bytes < req->rx_len)
+	return 0;
-		return;
+}
 
-	req_close(req, 0, NULL);
+
+static int req_recv(struct http_req *req, struct mbuf *mb, bool *last)
+{
+	int err;
+
+	*last = false;
+
+	if (!req->chunked) {
+
+		err = write_body(req, mb);
+		if (err)
+			return err;
+
+		if (req->rx_len == 0)
+			*last = true;
+
+		return 0;
+	}
+
+	while (mbuf_get_left(mb)) {
+
+		if (req->rx_len == 0) {
+
+			err = http_chunk_decode(&req->chunk, mb, &req->rx_len);
+			if (err == ENODATA)
+				return 0;
+			else if (err)
+				return err;
+			else if (req->rx_len == 0) {
+				*last = true;
+				return 0;
+			}
+		}
+
+		err = write_body(req, mb);
+		if (err)
+			return err;
+	}
+
+	return 0;
 }
 
 
@@ -237,18 +299,21 @@ static void estab_handler(void *arg)
 
 static void recv_handler(struct mbuf *mb, void *arg)
 {
-	struct http_msg *msg = NULL;
 	const struct http_hdr *hdr;
 	struct conn *conn = arg;
 	struct http_req *req = conn->req;
 	size_t pos;
+	bool last;
 	int err;
 
 	if (!req)
 		return;
 
-	if (req->data) {
+	if (req->msg) {
-		req_recv(req, mb);
+		err = req_recv(req, mb, &last);
+		if (err || last)
+			goto out;
+
 		return;
 	}
 
@@ -276,7 +341,7 @@ static void recv_handler(struct mbuf *mb, void *arg)
 
 	pos = req->mb->pos;
 
-	err = http_msg_decode(&msg, req->mb, false);
+	err = http_msg_decode(&req->msg, req->mb, false);
 	if (err) {
 		if (err == ENODATA) {
 			req->mb->pos = pos;
@@ -285,50 +350,27 @@ static void recv_handler(struct mbuf *mb, void *arg)
 		goto out;
 	}
 
-	hdr = http_msg_hdr(msg, HTTP_HDR_CONNECTION);
+	if (req->datah)
+		tmr_cancel(&conn->tmr);
 
+	hdr = http_msg_hdr(req->msg, HTTP_HDR_CONNECTION);
 	if (hdr && !pl_strcasecmp(&hdr->val, "close"))
 		req->close = true;
 
-	if (req->datah) {
+	if (http_msg_hdr_has_value(req->msg, HTTP_HDR_TRANSFER_ENCODING,
+				   "chunked"))
+		req->chunked = true;
+	else
+		req->rx_len = req->msg->clen;
 
-		uint32_t nrefs;
+	err = req_recv(req, req->mb, &last);
+	if (err || last)
+		goto out;
 
-		if (http_msg_hdr(msg, HTTP_HDR_CONTENT_LENGTH))
+	return;
-			req->rx_len = msg->clen;
-		else
-			req->rx_len = -1;
-
-		tmr_cancel(&conn->tmr);
-		req->data = true;
-
-		mem_ref(req);
-
-		if (req->resph)
-			req->resph(0, msg, req->arg);
-
-		nrefs = mem_nrefs(req);
-		mem_deref(req);
-
-		mem_deref(msg);
-
-		if (nrefs > 1 && mbuf_get_left(req->mb))
-			req_recv(req, req->mb);
-
-		return;
-	}
-
-	if (mbuf_get_left(req->mb) < msg->clen) {
-		req->mb->pos = pos;
-		mem_deref(msg);
-		return;
-	}
-
-	req->mb->end = req->mb->pos + msg->clen;
 
  out:
-	req_close(req, err, msg);
+	req_close(req, err, req->msg);
-	mem_deref(msg);
 }
 
 
@@ -600,6 +642,21 @@ int http_request(struct http_req **reqp, struct http_cli *cli, const char *met,
 }
 
 
+/**
+ * Set HTTP request connection handler
+ *
+ * @param req   HTTP request object
+ * @param connh Connection handler
+ */
+void http_req_set_conn_handler(struct http_req *req, http_conn_h *connh)
+{
+	if (!req)
+		return;
+
+	req->connh = connh;
+}
+
+
 /**
  * Allocate an HTTP client instance
  *
@@ -642,21 +699,3 @@ int http_client_alloc(struct http_cli **clip, struct dnsc *dnsc)
 
 	return err;
 }
-
-
-struct tcp_conn *http_req_tcp(struct http_req *req)
-{
-	if (!req || !req->conn)
-		return NULL;
-
-	return req->conn->tc;
-}
-
-
-struct tls_conn *http_req_tls(struct http_req *req)
-{
-	if (!req || !req->conn)
-		return NULL;
-
-	return req->conn->sc;
-}

src/http/http.h

@@ -0,0 +1,17 @@
+/**
+ * @file http.h  HTTP Private Interface
+ *
+ * Copyright (C) 2010 Creytiv.com
+ */
+
+
+struct http_chunk {
+	size_t size;
+	unsigned lf;
+	bool trailer;
+	bool digit;
+	bool param;
+};
+
+
+int http_chunk_decode(struct http_chunk *chunk, struct mbuf *mb, size_t *size);

src/http/mod.mk

@@ -5,6 +5,7 @@
 #
 
 SRCS	+= http/auth.c
+SRCS	+= http/chunk.c
 SRCS	+= http/client.c
 SRCS	+= http/msg.c
 SRCS	+= http/server.c

src/http/msg.c

@@ -32,6 +32,7 @@ static void destructor(void *arg)
 	struct http_msg *msg = arg;
 
 	list_flush(&msg->hdrl);
+	mem_deref(msg->_mb);
 	mem_deref(msg->mb);
 }
 
@@ -163,7 +164,13 @@ int http_msg_decode(struct http_msg **msgp, struct mbuf *mb, bool req)
 	if (!msg)
 		return ENOMEM;
 
-	msg->mb = mem_ref(mb);
+	msg->_mb = mem_ref(mb);
+
+	msg->mb = mbuf_alloc(8192);
+	if (!msg->mb) {
+		err = ENOMEM;
+		goto out;
+	}
 
 	if (req) {
 		if (re_regex(s.p, s.l, "[a-z]+ [^? ]+[^ ]* HTTP/[0-9.]+",

src/http/server.c

@@ -144,6 +144,9 @@ static void recv_handler(struct mbuf *mb, void *arg)
 			break;
 		}
 
+		mem_deref(msg->mb);
+		msg->mb = mem_ref(msg->_mb);
+
 		mb = conn->mb;
 
 		end     = mb->end;

src/websock/websock.c

@@ -412,13 +412,8 @@ static void http_resp_handler(int err, const struct http_msg *msg, void *arg)
 	/* here we are ok */
 
 	conn->state = OPEN;
-	conn->tc = mem_ref(http_req_tcp(conn->req));
-	conn->sc = mem_ref(http_req_tls(conn->req));
 	(void)tcp_conn_peer_get(conn->tc, &conn->peer);
 
-	tcp_set_handlers(conn->tc, NULL, recv_handler, close_handler, conn);
-	conn->req = mem_deref(conn->req);
-
 	if (conn->kaint)
 		tmr_start(&conn->tmr, conn->kaint, keepalive_handler, conn);
 
@@ -430,13 +425,15 @@ static void http_resp_handler(int err, const struct http_msg *msg, void *arg)
 }
 
 
-/* dummy HTTP data handler, this must be here so that HTTP client
+static void http_conn_handler(struct tcp_conn *tc, struct tls_conn *sc,
- * is not closing the underlying TCP-connection (which we need ..)
+			      void *arg)
- */
-static void http_data_handler(struct mbuf *mb, void *arg)
 {
-	(void)mb;
+	struct websock_conn *conn = arg;
-	(void)arg;
+
+	conn->tc = mem_ref(tc);
+	conn->sc = mem_ref(sc);
+
+	tcp_set_handlers(conn->tc, NULL, recv_handler, close_handler, conn);
 }
 
 
@@ -480,7 +477,7 @@ int websock_connect(struct websock_conn **connp, struct websock *sock,
 	/* Protocol Handshake */
 	va_start(ap, fmt);
 	err = http_request(&conn->req, cli, "GET", uri,
-			   http_resp_handler, http_data_handler, conn,
+			   http_resp_handler, NULL, conn,
 			   "Upgrade: websocket\r\n"
 			   "Connection: upgrade\r\n"
 			   "Sec-WebSocket-Key: %b\r\n"
@@ -493,6 +490,8 @@ int websock_connect(struct websock_conn **connp, struct websock *sock,
 	if (err)
 		goto out;
 
+	http_req_set_conn_handler(conn->req, http_conn_handler);
+
  out:
 	if (err)
 		mem_deref(conn);