diff --git a/CMakeLists.txt b/CMakeLists.txt index a02d66453..d30f49366 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -139,6 +139,8 @@ option(LWS_WITH_SECURE_STREAMS_PROXY_API "Secure Streams support to work across option(LWS_WITH_SECURE_STREAMS_SYS_AUTH_API_AMAZON_COM "Auth support for api.amazon.com" OFF) option(LWS_WITH_SECURE_STREAMS_STATIC_POLICY_ONLY "Secure Streams Policy is hardcoded only" OFF) option(LWS_WITH_SECURE_STREAMS_AUTH_SIGV4 "Secure Streams Auth support for AWS Sigv4" OFF) +option(LWS_WITH_SECURE_STREAMS_BUFFER_DUMP "Secure Streams protocol buffer dump" OFF) +option(LWS_WITH_SS_DIRECT_PROTOCOL_STR "Secure Streams directly set/get metadata w/o policy" OFF) # # CTest options diff --git a/cmake/lws_config.h.in b/cmake/lws_config.h.in index d0f81b9e8..6ea42574e 100644 --- a/cmake/lws_config.h.in +++ b/cmake/lws_config.h.in @@ -192,6 +192,8 @@ #cmakedefine LWS_WITH_SECURE_STREAMS_PROXY_API #cmakedefine LWS_WITH_SECURE_STREAMS_STATIC_POLICY_ONLY #cmakedefine LWS_WITH_SECURE_STREAMS_AUTH_SIGV4 +#cmakedefine LWS_WITH_SECURE_STREAMS_BUFFER_DUMP +#cmakedefine LWS_WITH_SS_DIRECT_PROTOCOL_STR #cmakedefine LWS_WITH_SELFTESTS #cmakedefine LWS_WITH_SEQUENCER #cmakedefine LWS_WITH_SERVER_STATUS diff --git a/include/libwebsockets/lws-secure-streams-policy.h b/include/libwebsockets/lws-secure-streams-policy.h index 55587e125..4b782ae6d 100644 --- a/include/libwebsockets/lws-secure-streams-policy.h +++ b/include/libwebsockets/lws-secure-streams-policy.h @@ -1,7 +1,7 @@ /* * libwebsockets - small server side websockets and web server implementation * - * Copyright (C) 2019 - 2020 Andy Green + * Copyright (C) 2019 - 2021 Andy Green * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -158,6 +158,9 @@ enum { /**< stream is not critical and should be handled as cheap as poss */ LWSSSPOLF_PERF = (1 << 22), /**< capture and report performace information */ + LWSSSPOLF_DIRECT_PROTO_STR = (1 << 23), + /**< metadata as direct protocol string, e.g. http header */ + }; typedef struct lws_ss_trust_store { @@ -197,6 +200,9 @@ typedef struct lws_ss_metadata { uint8_t value_length; /* only valid if set by policy */ uint8_t value_is_http_token; /* valid if set by policy */ +#if defined(LWS_WITH_SS_DIRECT_PROTOCOL_STR) + uint8_t name_on_lws_heap:1; /* proxy metatadata does this */ +#endif uint8_t value_on_lws_heap:1; /* proxy + rx metadata does this */ #if defined(LWS_WITH_SECURE_STREAMS_PROXY_API) uint8_t pending_onward:1; diff --git a/include/libwebsockets/lws-secure-streams.h b/include/libwebsockets/lws-secure-streams.h index 9a929cff1..47c8fb385 100644 --- a/include/libwebsockets/lws-secure-streams.h +++ b/include/libwebsockets/lws-secure-streams.h @@ -347,6 +347,11 @@ typedef lws_ss_state_return_t (*lws_sscb_state)(void *userobj, void *h_src, lws_ss_constate_t state, lws_ss_tx_ordinal_t ack); +#if defined(LWS_WITH_SECURE_STREAMS_BUFFER_DUMP) +typedef void (*lws_ss_buffer_dump_cb)(void *userobj, const uint8_t *buf, + size_t len, int done); +#endif + struct lws_ss_policy; typedef struct lws_ss_info { @@ -379,6 +384,10 @@ typedef struct lws_ss_info { /**< advisory cb about state of stream and QoS status if applicable... * h_src is only used with sinks and LWSSSCS_SINK_JOIN/_PART events. * Return nonzero to indicate you want to destroy the stream. */ +#if defined(LWS_WITH_SECURE_STREAMS_BUFFER_DUMP) + lws_ss_buffer_dump_cb dump; + /**< cb to record needed protocol buffer data*/ +#endif int manual_initial_tx_credit; /**< 0 = manage any tx credit automatically, nonzero explicitly sets the * peer stream to have the given amount of tx credit, if the protocol diff --git a/lib/roles/http/client/client-http.c b/lib/roles/http/client/client-http.c index 8730ee612..e09b6f01c 100644 --- a/lib/roles/http/client/client-http.c +++ b/lib/roles/http/client/client-http.c @@ -419,6 +419,21 @@ client_http_body_sent: } m = eb.len - n; +#if defined(LWS_WITH_SECURE_STREAMS_BUFFER_DUMP) + do { + lws_ss_handle_t *h = (lws_ss_handle_t *)lws_get_opaque_user_data(wsi); + if (!h) + break; + + if (h->info.dump) { + h->info.dump(ss_to_userobj(h), + (const uint8_t *)eb.token, + (size_t)m, + (wsi->http.ah->parser_state == + WSI_PARSING_COMPLETE) ? 1 : 0); + } + } while (0); +#endif if (lws_buflist_aware_finished_consuming(wsi, &eb, m, buffered, __func__)) diff --git a/lib/secure-streams/README.md b/lib/secure-streams/README.md index fcff2dcef..96978d7f3 100644 --- a/lib/secure-streams/README.md +++ b/lib/secure-streams/README.md @@ -380,11 +380,17 @@ Indicate that the streamtype should use the named auth type from the `auth` array in the policy ### `aws_region` + Indicate which metadata should be used to set aws region for certain streamtype ### `aws_service` + Indicate which metadata should be used to set aws service for certain streamtype +### `direct_proto_str` + +If set to `true`, application can use `lws_ss_set_metadata()` to directly set protocol related string and use `lws_ss_get_metadata` to fetch certain protocol related string. Please note that currently HTTP header is the supported protocol string. The `name` parameter is the name of HTTP header name (**with ':'**, e.g. `"Content-Type:"`) and `value` is the header's value. `LWS_WITH_SS_DIRECT_PROTOCOL_STR` flag needs to be configured during compilation for this. Currently it's only work for non-proxy case. + ### `server_cert` **SERVER ONLY**: subject to change... the name of the x.509 cert that is the diff --git a/lib/secure-streams/policy-common.c b/lib/secure-streams/policy-common.c index 9899b41d7..31808467e 100644 --- a/lib/secure-streams/policy-common.c +++ b/lib/secure-streams/policy-common.c @@ -1,7 +1,7 @@ /* * libwebsockets - small server side websockets and web server implementation * - * Copyright (C) 2019 - 2020 Andy Green + * Copyright (C) 2019 - 2021 Andy Green * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -85,12 +85,31 @@ lws_ss_set_metadata(struct lws_ss_handle *h, const char *name, { lws_ss_metadata_t *omd = lws_ss_get_handle_metadata(h, name); - if (!omd) { - lwsl_info("%s: unknown metadata %s\n", __func__, name); - return 1; - } + if (omd) + return _lws_ss_set_metadata(omd, name, value, len); - return _lws_ss_set_metadata(omd, name, value, len); +#if defined(LWS_WITH_SS_DIRECT_PROTOCOL_STR) + if (h->policy->flags & LWSSSPOLF_DIRECT_PROTO_STR) { + omd = lws_ss_get_handle_instant_metadata(h, name); + if (!omd) { + omd = lws_zalloc(sizeof(*omd), "imetadata"); + if (!omd) { + lwsl_err("%s OOM\n", __func__); + return 1; + } + omd->name = name; + omd->next = h->instant_metadata; + h->instant_metadata = omd; + } + omd->value__may_own_heap = (void *)value; + omd->value_length = (uint8_t)len; + + return 0; + } +#endif + + lwsl_info("%s: unknown metadata %s\n", __func__, name); + return 1; } int @@ -141,16 +160,56 @@ lws_ss_get_metadata(struct lws_ss_handle *h, const char *name, const void **value, size_t *len) { lws_ss_metadata_t *omd = lws_ss_get_handle_metadata(h, name); +#if defined(LWS_WITH_SS_DIRECT_PROTOCOL_STR) + int n; +#endif - if (!omd) { - lwsl_info("%s: unknown metadata %s\n", __func__, name); + if (omd) { + *value = omd->value__may_own_heap; + *len = omd->length; + + return 0; + } +#if defined(LWS_WITH_SS_DIRECT_PROTOCOL_STR) + if (!(h->policy->flags & LWSSSPOLF_DIRECT_PROTO_STR)) + goto bail; + + n = lws_http_string_to_known_header(name, strlen(name)); + if (n != LWS_HTTP_NO_KNOWN_HEADER) { + *len = (size_t)lws_hdr_total_length(h->wsi, n); + if (!*len) + goto bail; + *value = lws_hdr_simple_ptr(h->wsi, n); + if (!*value) + goto bail; + + return 0; + } +#if defined(LWS_WITH_CUSTOM_HEADERS) + n = lws_hdr_custom_length(h->wsi, (const char *)name, + (int)strlen(name)); + if (n <= 0) + goto bail; + *value = lwsac_use(&h->imd_ac, (size_t)(n+1), (size_t)(n+1)); + if (!*value) { + lwsl_err("%s ac OOM\n", __func__); return 1; } - - *value = omd->value__may_own_heap; - *len = omd->length; + if (lws_hdr_custom_copy(h->wsi, (char *)(*value), n+1, name, + (int)strlen(name))) { + /* waste n+1 bytes until ss is destryed */ + goto bail; + } + *len = (size_t)n; return 0; +#endif + +bail: +#endif + lwsl_info("%s: unknown metadata %s\n", __func__, name); + + return 1; } lws_ss_metadata_t * @@ -165,6 +224,24 @@ lws_ss_get_handle_metadata(struct lws_ss_handle *h, const char *name) return NULL; } +#if defined(LWS_WITH_SS_DIRECT_PROTOCOL_STR) +lws_ss_metadata_t * +lws_ss_get_handle_instant_metadata(struct lws_ss_handle *h, const char *name) +{ + lws_ss_metadata_t *imd = h->instant_metadata; + + while (imd) { + if (!strcmp(name, imd->name)) + return imd; + imd = imd->next; + } + + return NULL; +} + +#endif + + lws_ss_metadata_t * lws_ss_policy_metadata(const lws_ss_policy_t *p, const char *name) { diff --git a/lib/secure-streams/policy-json.c b/lib/secure-streams/policy-json.c index dc0b64393..da3a86e0e 100644 --- a/lib/secure-streams/policy-json.c +++ b/lib/secure-streams/policy-json.c @@ -120,6 +120,7 @@ static const char * const lejp_tokens_policy[] = { "s[].*.use_auth", "s[].*.aws_region", "s[].*.aws_service", + "s[].*.direct_proto_str", "s[].*", "auth[].name", "auth[].type", @@ -220,6 +221,7 @@ typedef enum { LSSPPT_USE_AUTH, LSSPPT_AWS_REGION, LSSPPT_AWS_SERVICE, + LSSPPT_DIRECT_PROTO_STR, LSSPPT_STREAMTYPES, LSSPPT_AUTH_NAME, LSSPPT_AUTH_TYPE, @@ -1025,6 +1027,12 @@ lws_ss_policy_parser_cb(struct lejp_ctx *ctx, char reason) reason == LEJPCB_VAL_TRUE; break; #endif + case LSSPPT_DIRECT_PROTO_STR: + if (reason == LEJPCB_VAL_TRUE) + a->curr[LTY_POLICY].p->flags |= + LWSSSPOLF_DIRECT_PROTO_STR; + break; + case LSSPPT_PROTOCOL: a->curr[LTY_POLICY].p->protocol = 0xff; diff --git a/lib/secure-streams/private-lib-secure-streams.h b/lib/secure-streams/private-lib-secure-streams.h index 8b0df5420..3ae5b52a1 100644 --- a/lib/secure-streams/private-lib-secure-streams.h +++ b/lib/secure-streams/private-lib-secure-streams.h @@ -79,6 +79,10 @@ typedef struct lws_ss_handle { #endif lws_ss_metadata_t *metadata; +#if defined(LWS_WITH_SS_DIRECT_PROTOCOL_STR) + lws_ss_metadata_t *instant_metadata; /**< for set instant metadata */ + struct lwsac *imd_ac; /**< for get custom header */ +#endif const lws_ss_policy_t *rideshare; #if defined(LWS_WITH_CONMON) @@ -467,6 +471,11 @@ lws_ss_get_handle_metadata(struct lws_ss_handle *h, const char *name); lws_ss_metadata_t * lws_ss_policy_metadata_index(const lws_ss_policy_t *p, size_t index); +#if defined(LWS_WITH_SS_DIRECT_PROTOCOL_STR) +lws_ss_metadata_t * +lws_ss_get_handle_instant_metadata(struct lws_ss_handle *h, const char *name); +#endif + lws_ss_metadata_t * lws_ss_policy_metadata(const lws_ss_policy_t *p, const char *name); diff --git a/lib/secure-streams/protocols/ss-h1.c b/lib/secure-streams/protocols/ss-h1.c index b41d015bc..dece05a40 100644 --- a/lib/secure-streams/protocols/ss-h1.c +++ b/lib/secure-streams/protocols/ss-h1.c @@ -271,6 +271,39 @@ lws_apply_metadata(lws_ss_handle_t *h, struct lws *wsi, uint8_t *buf, } +#if defined(LWS_WITH_SS_DIRECT_PROTOCOL_STR) +static int +lws_apply_instant_metadata(lws_ss_handle_t *h, struct lws *wsi, uint8_t *buf, + uint8_t **pp, uint8_t *end) +{ + lws_ss_metadata_t *imd = h->instant_metadata; + + while (imd) { + if (imd->name && imd->value__may_own_heap) { + lwsl_debug("%s add header %s %s %d\n", __func__, + imd->name, + (char *)imd->value__may_own_heap, + imd->value_length); + if (lws_add_http_header_by_name(wsi, + (const unsigned char *)imd->name, + (const unsigned char *)imd->value__may_own_heap, + imd->value_length, pp, end)) + return -1; + + /* it's possible user set content-length directly */ + if (!strncmp(imd->name, + "content-length", 14) && + atoi(imd->value__may_own_heap)) + lws_client_http_body_pending(wsi, 1); + + } + + imd = imd->next; + } + + return 0; +} +#endif /* * Check if any metadata headers present in the server headers, and record * them into the associated metadata item if so. @@ -717,6 +750,13 @@ malformed: if (lws_apply_metadata(h, wsi, buf, p, end)) return -1; +#if defined(LWS_WITH_SS_DIRECT_PROTOCOL_STR) + if (h->policy->flags & LWSSSPOLF_DIRECT_PROTO_STR) { + if (lws_apply_instant_metadata(h, wsi, buf, p, end)) + return -1; + } +#endif + #if defined(LWS_WITH_SECURE_STREAMS_AUTH_SIGV4) if (h->policy->auth && h->policy->auth->type && !strcmp(h->policy->auth->type, "sigv4")) { diff --git a/lib/secure-streams/secure-streams.c b/lib/secure-streams/secure-streams.c index e7f318e9b..b1c75e9b6 100644 --- a/lib/secure-streams/secure-streams.c +++ b/lib/secure-streams/secure-streams.c @@ -1332,6 +1332,23 @@ lws_ss_destroy(lws_ss_handle_t **ppss) pmd = pmd->next; } +#if defined(LWS_WITH_SS_DIRECT_PROTOCOL_STR) + { + + lws_ss_metadata_t *imd; + pmd = h->instant_metadata; + while (pmd) { + imd = pmd; + pmd = pmd->next; + lwsl_info("%s: instant md %p\n", __func__, imd); + lws_free(imd); + } + h->instant_metadata = NULL; + if (h->imd_ac) + lwsac_free(&h->imd_ac); + } +#endif + lws_sul_cancel(&h->sul); #if defined(LWS_WITH_SECURE_STREAMS_STATIC_POLICY_ONLY) diff --git a/minimal-examples/secure-streams/minimal-secure-streams/minimal-secure-streams.c b/minimal-examples/secure-streams/minimal-secure-streams/minimal-secure-streams.c index 6d48f04d3..3f4b2afb0 100644 --- a/minimal-examples/secure-streams/minimal-secure-streams/minimal-secure-streams.c +++ b/minimal-examples/secure-streams/minimal-secure-streams/minimal-secure-streams.c @@ -1,7 +1,7 @@ /* * lws-minimal-secure-streams * - * Written in 2010-2020 by Andy Green + * Written in 2010-2021 by Andy Green * * This file is made available under the Creative Commons CC0 1.0 * Universal Public Domain Dedication. @@ -118,7 +118,8 @@ static const char * const default_ss_policy = #endif "]," "\"s\": [" - /* +#if !defined(LWS_WITH_SS_DIRECT_PROTOCOL_STR) + /* * "fetch_policy" decides from where the real policy * will be fetched, if present. Otherwise the initial * policy is treated as the whole, hardcoded, policy. @@ -139,6 +140,25 @@ static const char * const default_ss_policy = "\"tls_trust_store\":" "\"le_via_dst\"," #endif "\"retry\":" "\"default\"" +#else + "{\"mintest\": {" + "\"endpoint\": \"warmcat.com\"," + "\"port\": 443," + "\"protocol\": \"h1\"," + "\"http_method\": \"GET\"," + "\"http_url\": \"index.html?uptag=${uptag}\"," + "\"metadata\": [{" + " \"uptag\": \"X-Upload-Tag:\"" + "}, {" + " \"xctype\": \"X-Content-Type:\"" + "}]," + "\"tls\": true," + "\"opportunistic\": true," + "\"retry\": \"default\"," + "\"timeout_ms\": 2000," + "\"direct_proto_str\": true," + "\"tls_trust_store\": \"le_via_dst\"" +#endif "}},{" /* * "captive_portal_detect" describes @@ -197,20 +217,23 @@ static const char *canned_root_token_payload = static lws_ss_state_return_t myss_rx(void *userobj, const uint8_t *buf, size_t len, int flags) { - myss_t *m = (myss_t *)userobj; - const char *md_srv = "not set", *md_test = "not set"; - size_t md_srv_len = 7, md_test_len = 7; if (flags & LWSSS_FLAG_PERF_JSON) return LWSSSSRET_OK; +#if !defined(LWS_WITH_SS_DIRECT_PROTOCOL_STR) + myss_t *m = (myss_t *)userobj; + const char *md_srv = "not set", *md_test = "not set"; + size_t md_srv_len = 7, md_test_len = 7; + lws_ss_get_metadata(m->ss, "srv", (const void **)&md_srv, &md_srv_len); lws_ss_get_metadata(m->ss, "test", (const void **)&md_test, &md_test_len); - lwsl_user("%s: len %d, flags: %d, srv: %.*s, test: %.*s\n", __func__, (int)len, flags, (int)md_srv_len, md_srv, (int)md_test_len, md_test); + lwsl_hexdump_info(buf, len); +#endif /* * If we received the whole message, for our example it means @@ -240,6 +263,21 @@ myss_state(void *userobj, void *sh, lws_ss_constate_t state, lws_ss_tx_ordinal_t ack) { myss_t *m = (myss_t *)userobj; +#if defined(LWS_WITH_SS_DIRECT_PROTOCOL_STR) + const char *md_test = "not set"; + size_t md_test_len = 7; + int i; + static const char * imd_test_keys[8] = { + "server:", + "content-security-policy:", + "strict-transport-security:", + "test-custom-header:", + "x-xss-protection:", + "x-content-type-options:", + "x-frame-options:", + "x-non-exist:", + }; +#endif lwsl_user("%s: %s (%d), ord 0x%x\n", __func__, lws_ss_state_name((int)state), state, (unsigned int)ack); @@ -254,10 +292,21 @@ myss_state(void *userobj, void *sh, lws_ss_constate_t state, if (lws_ss_set_metadata(m->ss, "uptag", "myuptag123", 10)) /* can fail, eg due to OOM, retry later if so */ return LWSSSSRET_DISCONNECT_ME; - +#if !defined(LWS_WITH_SS_DIRECT_PROTOCOL_STR) if (lws_ss_set_metadata(m->ss, "ctype", "myctype", 7)) /* can fail, eg due to OOM, retry later if so */ return LWSSSSRET_DISCONNECT_ME; +#else + if (lws_ss_set_metadata(m->ss, "X-Test-Type1:", "myctype1", 8)) + /* can fail, eg due to OOM, retry later if so */ + return LWSSSSRET_DISCONNECT_ME; + if (lws_ss_set_metadata(m->ss, "X-Test-Type2:", "myctype2", 8)) + /* can fail, eg due to OOM, retry later if so */ + return LWSSSSRET_DISCONNECT_ME; + if (lws_ss_set_metadata(m->ss, "Content-Type:", "myctype", 7)) + /* can fail, eg due to OOM, retry later if so */ + return LWSSSSRET_DISCONNECT_ME; +#endif break; case LWSSSCS_ALL_RETRIES_FAILED: @@ -265,6 +314,16 @@ myss_state(void *userobj, void *sh, lws_ss_constate_t state, interrupted = 1; bad = 2; break; + case LWSSSCS_CONNECTED: +#if defined(LWS_WITH_SS_DIRECT_PROTOCOL_STR) + lwsl_user("%s: get direct metadata\n", __func__); + for (i = 0; i < 8; i++) { + md_test = "not set"; + lws_ss_get_metadata(m->ss, imd_test_keys[i], (const void **)&md_test, &md_test_len); + lwsl_user("%s test key:[%s], got [%s]\n", __func__, imd_test_keys[i], md_test); + } +#endif + break; case LWSSSCS_QOS_ACK_REMOTE: lwsl_notice("%s: LWSSSCS_QOS_ACK_REMOTE\n", __func__); @@ -288,6 +347,15 @@ myss_state(void *userobj, void *sh, lws_ss_constate_t state, return LWSSSSRET_OK; } +#if defined(LWS_WITH_SECURE_STREAMS_BUFFER_DUMP) +static void +myss_headers_dump(void *userobj, const uint8_t *buf, size_t len, int done) +{ + lwsl_user("%s: %lu done: %s\n", __func__, len, done?"true":"false"); + + lwsl_hexdump_err(buf, len); +} +#endif static int app_system_state_nf(lws_state_manager_t *mgr, lws_state_notify_link_t *link, int current, int target) @@ -371,6 +439,9 @@ app_system_state_nf(lws_state_manager_t *mgr, lws_state_notify_link_t *link, ssi.rx = myss_rx; ssi.tx = myss_tx; ssi.state = myss_state; +#if defined(LWS_WITH_SECURE_STREAMS_BUFFER_DUMP) + ssi.dump = myss_headers_dump; +#endif ssi.user_alloc = sizeof(myss_t); ssi.streamtype = test_ots ? "mintest-ots" : (test_respmap ? "respmap" : "mintest"); @@ -434,6 +505,8 @@ int main(int argc, const char **argv) memset(&info, 0, sizeof info); lws_cmdline_option_handle_builtin(argc, argv, &info); + //lws_set_log_level(LLL_USER | LLL_ERR | LLL_DEBUG | LLL_NOTICE | LLL_INFO, NULL); + lwsl_user("LWS secure streams test client [-d]\n"); /* these options are mutually exclusive if given */