http: add RFC7231 date and time helpers and retry-after handling

Teach lws how to deal with date: and retry-after:

Add quick selftest into apt-test-lws_tokenize

Expand lws_retry_sul_schedule_retry_wsi() to check for retry_after and
increase the backoff if a larger one found.

Finally, change SS h1 protocol to handle 503 + retry-after: as a
failure, and apply any increased backoff from retry-after
automatically.

include/libwebsockets/lws-http.h

@@ -731,6 +731,53 @@ LWS_VISIBLE LWS_EXTERN int
 lws_urldecode(char *string, const char *escaped, int len);
 ///@}
 
+/**
+ * lws_http_date_render_from_unix() - render unixtime as RFC7231 date string
+ *
+ * \param buf:		Destination string buffer
+ * \param len:		avilable length of dest string buffer in bytes
+ * \param t:		pointer to the time_t to render
+ *
+ * Returns 0 if time_t is rendered into the string buffer successfully, else
+ * nonzero.
+ */
+LWS_VISIBLE LWS_EXTERN int
+lws_http_date_render_from_unix(char *buf, size_t len, const time_t *t);
+
+/**
+ * lws_http_date_parse_unix() - parse a RFC7231 date string into unixtime
+ *
+ * \param b:		Source string buffer
+ * \param len:		avilable length of source string buffer in bytes
+ * \param t:		pointer to the destination time_t to set
+ *
+ * Returns 0 if string buffer parsed as RFC7231 time successfully, and
+ * *t set to the parsed unixtime, else return nonzero.
+ */
+LWS_VISIBLE LWS_EXTERN int
+lws_http_date_parse_unix(const char *b, size_t len, time_t *t);
+
+/**
+ * lws_http_check_retry_after() - increase a timeout if retry-after present
+ *
+ * \param wsi:		http stream this relates to
+ * \param us_interval_in_out: default us retry interval on entry may be updated
+ *
+ * This function may extend the incoming retry interval if the server has
+ * requested that using retry-after: header.  It won't reduce the incoming
+ * retry interval, only leave it alone or increase it.
+ *
+ * *us_interval_in_out should be set to a default retry interval on entry, if
+ * the wsi has a retry-after time or interval that resolves to an interval
+ * longer than the entry *us_interval_in_out, that will be updated to the longer
+ * interval and return 0.
+ *
+ * If no usable retry-after or the time is now or in the past,
+ * *us_interval_in_out is left alone and the function returns nonzero.
+ */
+LWS_VISIBLE LWS_EXTERN int
+lws_http_check_retry_after(struct lws *wsi, lws_usec_t *us_interval_in_out);
+
 /**
  * lws_return_http_status() - Return simple http status
  * \param wsi:		Websocket instance (available from user callback)

lib/core-net/network.c

@@ -370,6 +370,8 @@ lws_socket_bind(struct lws_vhost *vhost, lws_sockfd_type sockfd, int port,
 	return port;
 }
 
+#if defined(LWS_WITH_CLIENT)
+
 unsigned int
 lws_retry_get_delay_ms(struct lws_context *context,
 		       const lws_retry_bo_t *retry, uint16_t *ctry,
@@ -430,10 +432,30 @@ int
 lws_retry_sul_schedule_retry_wsi(struct lws *wsi, lws_sorted_usec_list_t *sul,
 				 sul_cb_t cb, uint16_t *ctry)
 {
-	return lws_retry_sul_schedule(wsi->a.context, wsi->tsi, sul,
-				      wsi->retry_policy, cb, ctry);
+	char conceal;
+	lws_usec_t us = lws_retry_get_delay_ms(wsi->a.context,
+					       wsi->retry_policy, ctry,
+					       &conceal) * LWS_US_PER_MS;
+
+	if (!conceal)
+		/* if our reties are up, they're up... */
+		return 1;
+
+#if defined(LWS_ROLE_H1) || defined(LWS_ROLE_H2)
+	if (wsi->role_ops == &role_ops_h1 || wsi->role_ops == &role_ops_h2)
+		/*
+		 * Since we're doing it by wsi, we're in a position to check for
+		 * http retry-after, it will increase us accordingly if found
+		 */
+		lws_http_check_retry_after(wsi, &us);
+#endif
+	lws_sul_schedule(wsi->a.context, wsi->tsi, sul, cb, us);
+
+	return 0;
 }
 
+#endif
+
 #if defined(LWS_WITH_IPV6)
 unsigned long
 lws_get_addr_scope(const char *ifname_or_ipaddr)

lib/roles/http/CMakeLists.txt

@@ -33,6 +33,7 @@ include_directories(. ./compression)
 
 list(APPEND SOURCES
 	roles/http/header.c
+	roles/http/date.c
 	roles/http/parsers.c)
 	
 if (NOT LWS_WITHOUT_SERVER)
@@ -87,4 +88,4 @@ endif()
 # Keep explicit parent scope exports at end
 #
 
-exports_to_parent_scope()
+exports_to_parent_scope()

lib/roles/http/date.c

@@ -0,0 +1,209 @@
+/*
+ * libwebsockets - small server side websockets and web server implementation
+ *
+ * Copyright (C) 2010 - 2020 Andy Green <andy@warmcat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * RFC7231 date string generation and parsing
+ */
+
+#include "private-lib-core.h"
+
+/*
+ * To avoid needless pointers, we encode these in one string using the fact
+ * they're 3 chars each to index it
+ */
+
+static const char *const s =
+		"JanFebMarAprMayJunJulAugSepOctNovDecMonTueWedThuFriSatSun";
+
+static int
+lws_http_date_render(char *buf, size_t len, const struct tm *tm)
+{
+	const char *w = s + 36 + (3 * tm->tm_wday), *m = s + (3 * tm->tm_mon);
+
+	if (len < 29)
+		return -1;
+
+	lws_snprintf(buf, len, "%c%c%c, %02d %c%c%c %d %02d:%02d:%02d GMT",
+		     w[0], w[1], w[2], tm->tm_mday, m[0], m[1], m[2],
+		     1900 + tm->tm_year, tm->tm_hour, tm->tm_min, tm->tm_sec);
+
+	return 0;
+}
+
+
+int
+lws_http_date_render_from_unix(char *buf, size_t len, const time_t *t)
+{
+	struct tm *tm = gmtime(t);
+
+	if (lws_http_date_render(buf, len, tm))
+		return -1;
+
+	return 0;
+}
+
+static int
+lws_http_date_parse(const char *b, size_t len, struct tm *tm)
+{
+	int n;
+
+	if (len < 29)
+		return -1;
+
+	/*
+	 * We reject anything that isn't a properly-formatted RFC7231 date, eg
+	 *
+	 *     Tue, 15 Nov 1994 08:12:31 GMT
+	 */
+
+	if (b[3] != ','  || b[4] != ' '  || b[7] != ' '  || b[11] != ' ' ||
+	    b[16] != ' ' || b[19] != ':' || b[22] != ':' || b[25] != ' ' ||
+	    b[26] != 'G' || b[27] != 'M' || b[28] != 'T')
+		return -1;
+
+	memset(tm, 0, sizeof(*tm));
+
+	for (n = 36; n < 57; n += 3)
+		if (b[0] == s[n] && b[1] == s[n + 1] && b[2] == s[n + 2])
+			break;
+		else
+			tm->tm_wday++;
+
+	if (n == 57)
+		return -1;
+
+	for (n = 0; n < 36; n += 3)
+		if (b[8] == s[n] && b[9] == s[n + 1] && b[10] == s[n + 2])
+			break;
+		else
+			tm->tm_mon++;
+
+	if (n == 36)
+		return -1;
+
+	tm->tm_mday = atoi(b + 5);
+	n = atoi(b + 12);
+	if (n < 1900)
+		return -1;
+	tm->tm_year = n - 1900;
+
+	n = atoi(b + 17);
+	if (n < 0 || n > 23)
+		return -1;
+	tm->tm_hour = n;
+
+	n = atoi(b + 20);
+	if (n < 0 || n > 60)
+		return -1;
+	tm->tm_min = n;
+
+	n = atoi(b + 23);
+	if (n < 0 || n > 61) /* leap second */
+		return -1;
+	tm->tm_sec = n;
+
+	return 0;
+}
+
+int
+lws_http_date_parse_unix(const char *b, size_t len, time_t *t)
+{
+	struct tm tm;
+
+	if (lws_http_date_parse(b, len, &tm))
+		return -1;
+
+	*t = mktime(&tm);
+
+	return (int)*t == -1 ? -1 : 0;
+}
+
+#if defined(LWS_WITH_CLIENT)
+
+int
+lws_http_check_retry_after(struct lws *wsi, lws_usec_t *us_interval_in_out)
+{
+	size_t len = lws_hdr_total_length(wsi, WSI_TOKEN_HTTP_RETRY_AFTER);
+	char *p = lws_hdr_simple_ptr(wsi, WSI_TOKEN_HTTP_RETRY_AFTER);
+	lws_usec_t u;
+	time_t t, td;
+
+	if (!p)
+		return 1;
+
+	/*
+	 * There are two arg styles for RETRY_AFTER specified in RFC7231 7.1.3,
+	 * either a full absolute second-resolution date/time, or an integer
+	 * interval
+	 *
+	 *      Retry-After: Fri, 31 Dec 1999 23:59:59 GMT
+         *      Retry-After: 120
+	 */
+
+	if (len < 9)
+		u = ((lws_usec_t)(time_t)atoi(p)) * LWS_USEC_PER_SEC;
+	else {
+
+		if (lws_http_date_parse_unix(p, len, &t))
+			return 1;
+
+		/*
+		 * If possible, look for DATE from the server as well, so we
+		 * can calculate the interval it thinks it is giving us,
+		 * eliminating problems from server - client clock skew
+		 */
+
+		time(&td);
+		len = lws_hdr_total_length(wsi, WSI_TOKEN_HTTP_DATE);
+		if (len) {
+			p = lws_hdr_simple_ptr(wsi, WSI_TOKEN_HTTP_DATE);
+			/* if this fails, it leaves td as client time */
+			(void)lws_http_date_parse_unix(p, len, &td);
+		}
+
+		if (td >= t)
+			/*
+			 * if he's effectively giving us a 0 or negative
+			 * interval, just ignore the whole thing and keep the
+			 * incoming interval
+			 */
+			return 1;
+
+		u = ((lws_usec_t)(t - td)) * LWS_USEC_PER_SEC;
+	}
+
+	/*
+	 * We are only willing to increase the incoming interval, not
+	 * decrease it
+	 */
+
+	if (u < *us_interval_in_out)
+		/* keep the incoming interval */
+		return 1;
+
+	/* use the computed interval */
+	*us_interval_in_out = u;
+
+	return 0;
+}
+
+#endif

lib/roles/http/private-lib-roles-http.h

@@ -330,6 +330,12 @@ lws_http_multipart_headers(struct lws *wsi, uint8_t *p);
 int
 lws_http_string_to_known_header(const char *s, size_t slen);
 
+int
+lws_http_date_render_from_unix(char *buf, size_t len, const time_t *t);
+
+int
+lws_http_date_parse_unix(const char *b, size_t len, time_t *t);
+
 enum {
 	CCTLS_RETURN_ERROR		= -1,
 	CCTLS_RETURN_DONE		= 0,

lib/secure-streams/private-lib-secure-streams.h

@@ -390,6 +390,9 @@ lws_ss_sys_fetch_policy(struct lws_context *context);
 lws_ss_state_return_t
 lws_ss_event_helper(lws_ss_handle_t *h, lws_ss_constate_t cs);
 
+lws_ss_state_return_t
+_lws_ss_backoff(lws_ss_handle_t *h, lws_usec_t us_override);
+
 lws_ss_state_return_t
 lws_ss_backoff(lws_ss_handle_t *h);
 

lib/secure-streams/protocols/ss-h1.c

@@ -346,6 +346,7 @@ secstream_h1(struct lws *wsi, enum lws_callback_reasons reason, void *user,
 	lws_ss_state_return_t r;
 	int f = 0, m, status;
 	char conceal_eom = 0;
+	lws_usec_t inter;
 	size_t buflen;
 
 	switch (reason) {
@@ -410,6 +411,24 @@ secstream_h1(struct lws *wsi, enum lws_callback_reasons reason, void *user,
 			/* it's just telling use we connected / joined the nwsi */
 	//		break;
 
+		if (status == HTTP_STATUS_SERVICE_UNAVAILABLE /* 503 */ ) {
+			/*
+			 * We understand this attempt failed, and that we should
+			 * conceal this attempt.  If there's a specified
+			 * retry-after, we should use that if larger than our
+			 * computed backoff
+			 */
+
+			inter = 0;
+			lws_http_check_retry_after(wsi, &inter);
+
+			r = _lws_ss_backoff(h, inter);
+			if (r != LWSSSSRET_OK)
+				return _lws_ss_handle_state_ret(r, wsi, &h);
+
+			return -1; /* end this stream */
+		}
+
 		if (h->policy->u.http.resp_expect)
 			h->u.http.good_respcode =
 					status == h->policy->u.http.resp_expect;

lib/secure-streams/secure-streams.c

@@ -188,7 +188,7 @@ lws_ss_set_timeout_us(lws_ss_handle_t *h, lws_usec_t us)
 }
 
 lws_ss_state_return_t
-lws_ss_backoff(lws_ss_handle_t *h)
+_lws_ss_backoff(lws_ss_handle_t *h, lws_usec_t us_override)
 {
 	uint64_t ms;
 	char conceal;
@@ -208,14 +208,26 @@ lws_ss_backoff(lws_ss_handle_t *h)
 		return lws_ss_event_helper(h, LWSSSCS_ALL_RETRIES_FAILED);
 	}
 
-	h->seqstate = SSSEQ_RECONNECT_WAIT;
-	lws_ss_set_timeout_us(h, ms * LWS_US_PER_MS);
+	/* Only increase our planned backoff, or go with it */
 
-	lwsl_info("%s: ss %p: retry wait %"PRIu64"ms\n", __func__, h, ms);
+	if (us_override < (lws_usec_t)ms * LWS_US_PER_MS)
+		us_override = ms * LWS_US_PER_MS;
+
+	h->seqstate = SSSEQ_RECONNECT_WAIT;
+	lws_ss_set_timeout_us(h, us_override);
+
+	lwsl_info("%s: ss %p: retry wait %dms\n", __func__, h,
+						  (int)(us_override / 1000));
 
 	return LWSSSSRET_OK;
 }
 
+lws_ss_state_return_t
+lws_ss_backoff(lws_ss_handle_t *h)
+{
+	return _lws_ss_backoff(h, 0);
+}
+
 #if defined(LWS_WITH_SYS_SMD)
 
 /*

minimal-examples/api-tests/api-test-lws_tokenize/main.c

@@ -600,6 +600,35 @@ int main(int argc, const char **argv)
 		printf("\t}\n");
 	}
 
+#if defined(LWS_ROLE_H1) || defined(LWS_ROLE_H2)
+	{
+		time_t t;
+
+		if (lws_http_date_parse_unix("Tue, 15 Nov 1994 08:12:31 GMT", 29, &t)) {
+			lwsl_err("%s: date parse failed\n", __func__);
+			fail++;
+		} else {
+			/* lwsl_notice("%s: %llu\n", __func__, (unsigned long long)t); */
+			if (t != (time_t)784887151) {
+				lwsl_err("%s: date parse wrong\n", __func__);
+				fail++;
+			} else {
+				char s[30];
+
+				if (lws_http_date_render_from_unix(s, sizeof(s), &t)) {
+					lwsl_err("%s: failed date render\n", __func__);
+					fail++;
+				} else {
+					if (!strcmp(s, "Tue, 15 Nov 1994 08:12:31 GMT")) {
+						lwsl_err("%s: date render wrong\n", __func__);
+						fail++;
+					}
+				}
+			}
+		}
+	}
+#endif
+
 	lwsl_user("Completed: PASS: %d, FAIL: %d\n", ok, fail);
 
 	return !(ok && !fail);