HTTP client improvements (#36)

* http client connection reuse

* retry failed requests using fresh connections

* fix line length

* Handle Connection: close response header
This commit is contained in:
Richard Aas 2017-01-05 21:53:32 +01:00 committed by Alfred E. Heggestad
parent 1ef18d6dc8
commit e167b2c154

View file

@ -10,6 +10,7 @@
#include <re_mbuf.h>
#include <re_sa.h>
#include <re_list.h>
#include <re_hash.h>
#include <re_fmt.h>
#include <re_tmr.h>
#include <re_srtp.h>
@ -22,43 +23,75 @@
enum {
CONN_TIMEOUT = 30000,
RECV_TIMEOUT = 30000,
RECV_TIMEOUT = 60000,
IDLE_TIMEOUT = 900000,
BUFSIZE_MAX = 524288,
CONN_BSIZE = 256,
};
struct http_cli {
struct list reql;
struct hash *ht_conn;
struct dnsc *dnsc;
struct tls *tls;
};
struct conn;
struct http_req {
struct sa srvv[16];
struct tmr tmr;
struct le le;
struct http_req **reqp;
struct http_cli *cli;
struct dns_query *dq;
struct tls_conn *sc;
struct tcp_conn *tc;
struct conn *conn;
struct mbuf *mbreq;
struct mbuf *mb;
struct tls *tls;
char *host;
http_resp_h *resph;
http_data_h *datah;
void *arg;
size_t rx_bytes;
size_t rx_len;
unsigned srvc;
uint16_t port;
bool secure;
bool close;
bool data;
};
struct conn {
struct tmr tmr;
struct sa addr;
struct le he;
struct http_req *req;
struct tls_conn *sc;
struct tcp_conn *tc;
uint64_t usec;
};
static void req_close(struct http_req *req, int err,
const struct http_msg *msg);
static int req_connect(struct http_req *req);
static void timeout_handler(void *arg);
static void cli_destructor(void *arg)
{
struct http_cli *cli = arg;
struct le *le = cli->reql.head;
while (le) {
struct http_req *req = le->data;
le = le->next;
req_close(req, ECONNABORTED, NULL);
}
hash_flush(cli->ht_conn);
mem_deref(cli->ht_conn);
mem_deref(cli->dnsc);
mem_deref(cli->tls);
}
@ -68,26 +101,49 @@ static void req_destructor(void *arg)
{
struct http_req *req = arg;
tmr_cancel(&req->tmr);
list_unlink(&req->le);
mem_deref(req->dq);
mem_deref(req->sc);
mem_deref(req->tc);
mem_deref(req->conn);
mem_deref(req->mbreq);
mem_deref(req->mb);
mem_deref(req->tls);
mem_deref(req->host);
}
static void conn_destructor(void *arg)
{
struct conn *conn = arg;
tmr_cancel(&conn->tmr);
hash_unlink(&conn->he);
mem_deref(conn->sc);
mem_deref(conn->tc);
}
static void conn_idle(struct conn *conn)
{
tmr_start(&conn->tmr, IDLE_TIMEOUT, timeout_handler, conn);
conn->req = NULL;
}
static void req_close(struct http_req *req, int err,
const struct http_msg *msg)
{
tmr_cancel(&req->tmr);
list_unlink(&req->le);
req->dq = mem_deref(req->dq);
req->sc = mem_deref(req->sc);
req->tc = mem_deref(req->tc);
req->datah = NULL;
if (req->conn) {
if (err || req->close)
mem_deref(req->conn);
else
conn_idle(req->conn);
req->conn = NULL;
}
if (req->reqp) {
*req->reqp = NULL;
req->reqp = NULL;
@ -97,15 +153,27 @@ static void req_close(struct http_req *req, int err,
req->resph(err, msg, req->arg);
req->resph = NULL;
}
mem_deref(req);
}
static void timeout_handler(void *arg)
static void try_next(struct conn *conn, int err)
{
struct http_req *req = arg;
int err = ETIMEDOUT;
struct http_req *req = conn->req;
bool retry = conn->usec > 1;
if (req->srvc > 0) {
mem_deref(conn);
if (!req)
return;
req->conn = NULL;
if (retry)
++req->srvc;
if (req->srvc > 0 && !req->data) {
err = req_connect(req);
if (!err)
@ -113,36 +181,74 @@ static void timeout_handler(void *arg)
}
req_close(req, err, NULL);
}
static void req_recv(struct http_req *req, struct mbuf *mb)
{
uint32_t nrefs;
req->rx_bytes += mbuf_get_left(mb);
mem_ref(req);
if (req->datah)
req->datah(mb, req->arg);
nrefs = mem_nrefs(req);
mem_deref(req);
if (nrefs == 1)
return;
if (req->rx_bytes < req->rx_len)
return;
req_close(req, 0, NULL);
}
static void timeout_handler(void *arg)
{
struct conn *conn = arg;
try_next(conn, ETIMEDOUT);
}
static void estab_handler(void *arg)
{
struct http_req *req = arg;
struct conn *conn = arg;
struct http_req *req = conn->req;
int err;
err = tcp_send(req->tc, req->mbreq);
if (!req)
return;
err = tcp_send(conn->tc, req->mbreq);
if (err) {
req_close(req, err, NULL);
mem_deref(req);
try_next(conn, err);
return;
}
tmr_start(&req->tmr, RECV_TIMEOUT, timeout_handler, req);
tmr_start(&conn->tmr, RECV_TIMEOUT, timeout_handler, conn);
}
static void recv_handler(struct mbuf *mb, void *arg)
{
struct http_msg *msg = NULL;
struct http_req *req = arg;
const struct http_hdr *hdr;
struct conn *conn = arg;
struct http_req *req = conn->req;
size_t pos;
int err;
if (!req)
return;
if (req->data) {
if (req->datah)
req->datah(mb, req->arg);
req_recv(req, mb);
return;
}
@ -179,12 +285,36 @@ static void recv_handler(struct mbuf *mb, void *arg)
goto out;
}
hdr = http_msg_hdr(msg, HTTP_HDR_CONNECTION);
if (hdr && !pl_strcasecmp(&hdr->val, "close"))
req->close = true;
if (req->datah) {
tmr_cancel(&req->tmr);
uint32_t nrefs;
if (http_msg_hdr(msg, HTTP_HDR_CONTENT_LENGTH))
req->rx_len = msg->clen;
else
req->rx_len = -1;
tmr_cancel(&conn->tmr);
req->data = true;
mem_ref(req);
if (req->resph)
req->resph(0, msg, req->arg);
nrefs = mem_nrefs(req);
mem_deref(req);
mem_deref(msg);
if (nrefs > 1 && mbuf_get_left(req->mb))
req_recv(req, req->mb);
return;
}
@ -198,24 +328,91 @@ static void recv_handler(struct mbuf *mb, void *arg)
out:
req_close(req, err, msg);
mem_deref(req);
mem_deref(msg);
}
static void close_handler(int err, void *arg)
{
struct http_req *req = arg;
struct conn *conn = arg;
if (req->srvc > 0 && !req->data) {
try_next(conn, err ? err : ECONNRESET);
}
err = req_connect(req);
if (!err)
return;
static bool conn_cmp(struct le *le, void *arg)
{
const struct conn *conn = le->data;
const struct http_req *req = arg;
if (!sa_cmp(&req->srvv[req->srvc], &conn->addr, SA_ALL))
return false;
if (req->secure != !!conn->sc)
return false;
return conn->req == NULL;
}
static int conn_connect(struct http_req *req)
{
const struct sa *addr = &req->srvv[req->srvc];
struct conn *conn;
int err;
conn = list_ledata(hash_lookup(req->cli->ht_conn,
sa_hash(addr, SA_ALL), conn_cmp, req));
if (conn) {
err = tcp_send(conn->tc, req->mbreq);
if (!err) {
tmr_start(&conn->tmr, RECV_TIMEOUT,
timeout_handler, conn);
req->conn = conn;
conn->req = req;
++conn->usec;
return 0;
}
mem_deref(conn);
}
req_close(req, err ? err : ECONNRESET, NULL);
mem_deref(req);
conn = mem_zalloc(sizeof(*conn), conn_destructor);
if (!conn)
return ENOMEM;
hash_append(req->cli->ht_conn, sa_hash(addr, SA_ALL), &conn->he, conn);
conn->addr = *addr;
conn->usec = 1;
err = tcp_connect(&conn->tc, addr, estab_handler, recv_handler,
close_handler, conn);
if (err)
goto out;
#ifdef USE_TLS
if (req->secure) {
err = tls_start_tcp(&conn->sc, req->cli->tls, conn->tc, 0);
if (err)
goto out;
}
#endif
tmr_start(&conn->tmr, CONN_TIMEOUT, timeout_handler, conn);
req->conn = conn;
conn->req = req;
out:
if (err)
mem_deref(conn);
return err;
}
@ -227,29 +424,11 @@ static int req_connect(struct http_req *req)
--req->srvc;
tmr_cancel(&req->tmr);
req->sc = mem_deref(req->sc);
req->tc = mem_deref(req->tc);
req->mb = mem_deref(req->mb);
err = tcp_connect(&req->tc, &req->srvv[req->srvc],
estab_handler, recv_handler, close_handler,
req);
if (err)
continue;
#ifdef USE_TLS
if (req->secure) {
err = tls_start_tcp(&req->sc, req->tls, req->tc, 0);
if (err) {
req->tc = mem_deref(req->tc);
continue;
}
}
#endif
tmr_start(&req->tmr, CONN_TIMEOUT, timeout_handler, req);
break;
err = conn_connect(req);
if (!err)
break;
}
return err;
@ -303,7 +482,6 @@ static void query_handler(int err, const struct dnshdr *hdr, struct list *ansl,
fail:
req_close(req, err, NULL);
mem_deref(req);
}
@ -332,7 +510,7 @@ int http_request(struct http_req **reqp, struct http_cli *cli, const char *met,
va_list ap;
int err;
if (!reqp || !cli || !met || !uri)
if (!cli || !met || !uri)
return EINVAL;
if (re_regex(uri, strlen(uri), "[a-z]+://[^:/]+[:]*[0-9]*[^]+",
@ -358,7 +536,9 @@ int http_request(struct http_req **reqp, struct http_cli *cli, const char *met,
if (!req)
return ENOMEM;
req->tls = mem_ref(cli->tls);
list_append(&cli->reql, &req->le, req);
req->cli = cli;
req->secure = secure;
req->port = pl_isset(&port) ? pl_u32(&port) : defport;
req->resph = resph;
@ -411,7 +591,7 @@ int http_request(struct http_req **reqp, struct http_cli *cli, const char *met,
out:
if (err)
mem_deref(req);
else {
else if (reqp) {
req->reqp = reqp;
*reqp = req;
}
@ -440,6 +620,10 @@ int http_client_alloc(struct http_cli **clip, struct dnsc *dnsc)
if (!cli)
return ENOMEM;
err = hash_alloc(&cli->ht_conn, CONN_BSIZE);
if (err)
goto out;
#ifdef USE_TLS
err = tls_alloc(&cli->tls, TLS_METHOD_SSLV23, NULL, NULL);
#else
@ -462,11 +646,17 @@ int http_client_alloc(struct http_cli **clip, struct dnsc *dnsc)
struct tcp_conn *http_req_tcp(struct http_req *req)
{
return req ? req->tc : NULL;
if (!req || !req->conn)
return NULL;
return req->conn->tc;
}
struct tls_conn *http_req_tls(struct http_req *req)
{
return req ? req->sc : NULL;
if (!req || !req->conn)
return NULL;
return req->conn->sc;
}