diff --git a/CMakeLists.txt b/CMakeLists.txt index 5221cd08e..e39373bf3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -133,6 +133,7 @@ if (LWS_IPV6) else() option(LWS_WITH_RFC6724 "Enable RFC6724 DNS result sorting" OFF) endif() +option(LWS_WITH_SYS_FAULT_INJECTION "Enable fault injection support" OFF) # # Secure Streams diff --git a/READMEs/README.fault-injection.md b/READMEs/README.fault-injection.md new file mode 100644 index 000000000..d2b21245d --- /dev/null +++ b/READMEs/README.fault-injection.md @@ -0,0 +1,92 @@ +# `lws_fi` Fault Injection + +To provide better quality there's a need to not just test the code paths for +normal operation, but also that it acts correctly under various fault +conditions that may be difficult to arrange at test-time. + +Code handling the failures may be anywhere including during early initialization +or in user code before lws intialization. + +To help with this lws has `LWS_WITH_SYS_FAULT_INJECTION` build option that +provides a simple but powerful api for fault injection in any lws or user code. + +## Fault contexts and faults + +`lws_fi_t` objects represent a named fault injection rules, just in terms of +whether and how often to inject the fault. + +`lws_fi_ctx_t` objects are linked-lists of `lws_fi_t` objects. When Fault +Injection is enabled at build-time, the key system objects like the +`lws_context`, `lws_vhost`, `wsi` and Secure Stream handles / SSPC handles +contain their own `lws_fi_ctx_t` lists that may have any number of `lws_fi_t` +added to them. + +`lws_fi_ctx_t` objects are hierarchical, if a named rule is not found in, eg, +a wsi Fault injection context, then the vhost and finally the lws_context Fault +Injection contexts are searched for it before giving up. This allows for both +global and individual overridden Fault Injection rules at each level. + +## Integrating fault injection conditionals into code + +A simple api `lws_fi(fi_ctx, "name")` is provided that returns 0 if no fault to +be injected, or 1 if the fault should be synthesized. If there is no rule +matching "name", the answer is always to not inject a fault, ie, returns 0. + +By default then just enabling Fault Injection at build does not have any impact +on code operation since the user must first add the fault injection rules he +wants. + +The api keeps track of each time the context was asked and uses this information +to drive the decision about when to say yes, according to the type of rule + +|Injection rule type|Description| +|---|---| +|`LWSFI_ALWAYS`|Unconditionally inject the fault| +|`LWSFI_DETERMINISTIC`|after `pre` times without the fault, the next `count` times exhibit the fault`| +|`LWSFI_PROBABILISTIC`|exhibit a fault `pre` percentage of the time| +|`LWSFI_PATTERN`|Reference `pre` bits pointed to by `pattern` and fault if the bit set| + +## Addings Fault Injection Rules to `lws_fi_ctx_t` + +User code should prepare a `lws_fi_ctx_t` cleared down to zero if necessary, +and one of these, eg on the stack + +``` +typedef struct lws_fi { + const char *name; + uint8_t *pattern; + uint64_t pre__prob1; + uint64_t count__prob2; + char type; /* LWSFI_* */ +} lws_fi_t; +``` + +and call `lws_fi_add(lws_fi_ctx_t *fic, const lws_fi_t *fi);`, this will +allocate and copy the provided `fi` into the allocation, and attach it to +the `lws_fi_ctx_t` list. + +The creation info struct associated with the context, vhost, wsi or Secure +Stream has a `*fi` pointer you can set to your `lws_fi_ctx_t`, when creating +the object it will take ownership of any `lws_fi_t` you attached to it. + +So the `lws_fi_ctx_t` and the `lws_fi_t` used as a template for adding the +rules may be on the stack and safely and go out of scope after the object +creation api is called. The `lws_fi_t` `name` is also copied into the +allocation and does not need to continue to exist after it is added to the +`lws_fi_ctx_t`. The only exception is the `pattern` member if used, the +array pointed to is not copied and must exist for the lifetime of the rule. + +## Passing in fault injection rules + +A key requirement is that Fault Injection rules must be availble to the code +creating an object before the object has been created. This is why the user +code prepares a temporary context listing his rules, and offers it as part of +the creation info struct, rather than waiting for the object to be created and +then attach Fault Injection rules... it's too late to test faults during the +creation by then otherwise. + +## Using the namespace to target specific instances + +Wsi client connection can directly have fault injection objects attached to it +at client connection creation time. + diff --git a/cmake/lws_config.h.in b/cmake/lws_config.h.in index 732767c99..3d4b867da 100644 --- a/cmake/lws_config.h.in +++ b/cmake/lws_config.h.in @@ -193,8 +193,9 @@ #cmakedefine LWS_WITH_STRUCT_JSON #cmakedefine LWS_WITH_SUL_DEBUGGING #cmakedefine LWS_WITH_SQLITE3 -#cmakedefine LWS_WITH_SYS_NTPCLIENT #cmakedefine LWS_WITH_SYS_DHCP_CLIENT +#cmakedefine LWS_WITH_SYS_FAULT_INJECTION +#cmakedefine LWS_WITH_SYS_NTPCLIENT #cmakedefine LWS_WITH_SYS_STATE #cmakedefine LWS_WITH_THREADPOOL #cmakedefine LWS_WITH_TLS diff --git a/include/libwebsockets.h b/include/libwebsockets.h index bf027a6c6..ae4ab6b38 100644 --- a/include/libwebsockets.h +++ b/include/libwebsockets.h @@ -566,6 +566,7 @@ struct lws_vhost; struct lws; #include +#include #include #if defined(LWS_WITH_SYS_SMD) #include diff --git a/include/libwebsockets/lws-client.h b/include/libwebsockets/lws-client.h index 48cf21db2..ee30e2cd8 100644 --- a/include/libwebsockets/lws-client.h +++ b/include/libwebsockets/lws-client.h @@ -202,6 +202,12 @@ struct lws_client_connect_info { void *mqtt_cp; #endif +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + lws_fi_ctx_t *fi; + /**< Attach external Fault Injection context to the client wsi, + * hierarchy is wsi -> vhost -> context */ +#endif + uint16_t keep_warm_secs; /**< 0 means 5s. If the client connection to the endpoint becomes idle, * defer closing it for this many seconds in case another outgoing diff --git a/include/libwebsockets/lws-context-vhost.h b/include/libwebsockets/lws-context-vhost.h index dbde666dc..b90876a01 100644 --- a/include/libwebsockets/lws-context-vhost.h +++ b/include/libwebsockets/lws-context-vhost.h @@ -810,6 +810,16 @@ struct lws_context_creation_info { * to make disappear, in order to simulate and test udp retry flow */ #endif +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + lws_fi_ctx_t *fi; + /**< CONTEXT | VHOST: attach external Fault Injection context to the + * lws_context or vhost. If creating the context + default vhost in + * one step, only the context binds to \p fi. When creating a vhost + * otherwise this can bind to the vhost so the faults can be injected + * from the start. + */ +#endif + #if defined(LWS_WITH_SYS_SMD) lws_smd_notification_cb_t early_smd_cb; /**< CONTEXT: NULL, or an smd notification callback that will be registered @@ -830,6 +840,7 @@ struct lws_context_creation_info { * (20 for FREERTOS) */ #endif + /* Add new things just above here ---^ * This is part of the ABI, don't needlessly break compatibility * diff --git a/include/libwebsockets/lws-fault-injection.h b/include/libwebsockets/lws-fault-injection.h new file mode 100644 index 000000000..8f46f0efe --- /dev/null +++ b/include/libwebsockets/lws-fault-injection.h @@ -0,0 +1,115 @@ +/* + * libwebsockets - small server side websockets and web server implementation + * + * Copyright (C) 2010 - 2021 Andy Green + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + * Fault injection api if built with LWS_WITH_SYS_FAULT_INJECTION + */ + +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + +enum { + LWSFI_ALWAYS, + LWSFI_DETERMINISTIC, /* do .count injections after .pre then stop */ + LWSFI_PROBABILISTIC, /* .prob1 / .prob2 chance of injection */ + LWSFI_PATTERN, /* use .count bits in .pattern after .pre */ +}; + +typedef struct lws_fi { + const char *name; + const uint8_t *pattern; + uint64_t pre; + uint64_t count; + uint64_t times; /* start at 0, tracks usage */ + char type; /* LWSFI_* */ +} lws_fi_t; + +typedef struct lws_fi_ctx { + lws_dll2_owner_t fi_owner; + const char *name; + struct lws_fi_ctx *parent; +} lws_fi_ctx_t; + +/** + * lws_fi() - find out if we should perform the named fault injection this time + * + * \param fic: fault injection tracking context + * \param fi_name: name of fault injection + * + * This checks if the named fault is configured in the fi tracking context + * provided, if it is, then it will make a decision if the named fault should + * be applied this time, using the tracking in the named lws_fi_t. + * + * If the provided context has a parent, that is also checked for the named fi + * item recursively, with the first found being used to determine if to inject + * or not. + * + * If LWS_WITH_SYS_FAULT_INJECTION is not defined, then this always return 0. + */ +LWS_VISIBLE LWS_EXTERN int +lws_fi(lws_fi_ctx_t *fic, const char *fi_name); + +/** + * lws_fi_add() - add an allocated copy of fault injection to a context + * + * \param fic: fault injection tracking context + * \param fi: the fault injection details + * + * This allocates a copy of \p fi and attaches it to the fault injection context + * \p fic. + */ +LWS_VISIBLE LWS_EXTERN int +lws_fi_add(lws_fi_ctx_t *fic, const lws_fi_t *fi); + +/** + * lws_fi_remove() - remove an allocated copy of fault injection from a context + * + * \param fic: fault injection tracking context + * \param name: the fault injection name to remove + * + * This looks for the named fault injection and removes and destroys it from + * the specified fault injection context + */ +LWS_VISIBLE LWS_EXTERN void +lws_fi_remove(lws_fi_ctx_t *fic, const char *name); + +/** + * lws_fi_destroy() - removes all allocated fault injection entries + * + * \param fic: fault injection tracking context + * + * This walks any allocated fault injection entries in \p fic and detaches and + * destroys them. It doesn't try to destroc \p fic itself, since this is + * not usually directly allocated. + */ +LWS_VISIBLE LWS_EXTERN void +lws_fi_destroy(lws_fi_ctx_t *fic); + +#else + +/* + * Helper so we can leave lws_fi() calls embedded in the code being tested, + * if fault injection is not enabled then it just always says "no" at buildtime. + */ + +#define lws_fi(_fi_name, _fic) (0) + +#endif diff --git a/include/libwebsockets/lws-secure-streams.h b/include/libwebsockets/lws-secure-streams.h index bc1ba88d1..a23f611e3 100644 --- a/include/libwebsockets/lws-secure-streams.h +++ b/include/libwebsockets/lws-secure-streams.h @@ -353,6 +353,12 @@ typedef struct lws_ss_info { * connection instead of a named streamtype */ #endif +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + lws_fi_ctx_t *fi; + /**< Attach external Fault Injection context to the stream, hierarchy + * is ss->context */ +#endif + lws_sscb_rx rx; /**< callback with rx payload for this stream */ lws_sscb_tx tx; diff --git a/lib/core-net/client/connect.c b/lib/core-net/client/connect.c index 26763d179..dd3ca3516 100644 --- a/lib/core-net/client/connect.c +++ b/lib/core-net/client/connect.c @@ -87,6 +87,7 @@ lws_client_connect_via_info(const struct lws_client_connect_info *i) const struct lws_protocols *p; const char *cisin[CIS_COUNT]; int tid = 0, n, tsi = 0; + struct lws_vhost *vh; size_t size; char *pc; @@ -137,6 +138,29 @@ lws_client_connect_via_info(const struct lws_client_connect_info *i) if (wsi == NULL) goto bail; + vh = i->vhost; + if (!vh) { + vh = i->context->vhost_list; + + if (!vh) { /* coverity */ + lwsl_err("%s: no vhost\n", __func__); + goto bail; + } + if (!strcmp(vh->name, "system")) + vh = vh->vhost_next; + } + +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + wsi->fi.name = "wsi"; + wsi->fi.parent = &vh->fi; + if (i->fi) + /* + * This moves all the lws_fi_t from info->fi to the vhost fi, + * leaving it empty + */ + lws_fi_import(&wsi->fi, i->fi); +#endif + #if defined(LWS_WITH_DETAILED_LATENCY) && LWS_MAX_SMP > 1 wsi->detlat.tsi = tsi; #endif diff --git a/lib/core-net/close.c b/lib/core-net/close.c index 24e1d547b..fe01dff51 100644 --- a/lib/core-net/close.c +++ b/lib/core-net/close.c @@ -793,6 +793,10 @@ __lws_close_free_wsi_final(struct lws *wsi) } #endif +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + lws_fi_destroy(&wsi->fi); +#endif + __lws_wsi_remove_from_sul(wsi); sanity_assert_no_wsi_traces(wsi->a.context, wsi); __lws_free_wsi(wsi); diff --git a/lib/core-net/private-lib-core-net.h b/lib/core-net/private-lib-core-net.h index 8086f0c59..b7356d4b6 100644 --- a/lib/core-net/private-lib-core-net.h +++ b/lib/core-net/private-lib-core-net.h @@ -523,6 +523,11 @@ struct lws_vhost { struct lws_conn_stats conn_stats; #endif +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + lws_fi_ctx_t fi; + /**< Fault Injection ctx for the vhost, hierarchy vhost->context */ +#endif + uint64_t options; struct lws_context *context; @@ -740,6 +745,11 @@ struct lws { * deleted as they are tried, list empty == everything tried */ #endif +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + lws_fi_ctx_t fi; + /**< Fault Injection ctx for the wsi, hierarchy wsi->vhost->context */ +#endif + lws_sockaddr46 sa46_local; lws_sockaddr46 sa46_peer; diff --git a/lib/core-net/vhost.c b/lib/core-net/vhost.c index 095f6acb9..2e75439c8 100644 --- a/lib/core-net/vhost.c +++ b/lib/core-net/vhost.c @@ -573,6 +573,17 @@ lws_create_vhost(struct lws_context *context, else vh->name = info->vhost_name; +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + vh->fi.name = "vh"; + vh->fi.parent = &context->fi; + if (info->fi) + /* + * This moves all the lws_fi_t from info->fi to the vhost fi, + * leaving it empty + */ + lws_fi_import(&vh->fi, info->fi); +#endif + __lws_lc_tag(&context->lcg[LWSLCG_VHOST], &vh->lc, "%s|%s|%d", vh->name, info->iface ? info->iface : "", info->port); @@ -1390,6 +1401,10 @@ __lws_vhost_destroy2(struct lws_vhost *vh) lws_dll2_remove(&vh->vh_being_destroyed_list); +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + lws_fi_destroy(&vh->fi); +#endif + __lws_lc_untag(&vh->lc); memset(vh, 0, sizeof(*vh)); lws_free(vh); diff --git a/lib/core/context.c b/lib/core/context.c index 3576fc3c1..e556953ea 100644 --- a/lib/core/context.c +++ b/lib/core/context.c @@ -613,6 +613,17 @@ lws_create_context(const struct lws_context_creation_info *info) context->system_ops = info->system_ops; context->pt_serv_buf_size = (unsigned int)s1; +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + context->fi.name = "ctx"; + if (info->fi) + /* + * This moves all the lws_fi_t from info->fi to the context fi, + * leaving it empty, so no injection added to default vhost + */ + lws_fi_import(&context->fi, info->fi); +#endif + + #if defined(LWS_WITH_SYS_SMD) context->smd_ttl_us = info->smd_ttl_us ? info->smd_ttl_us : #if defined(LWS_PLAT_FREERTOS) @@ -1988,6 +1999,10 @@ next: NULL, NULL); #endif +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + lws_fi_destroy(&context->fi); +#endif + lws_free(context); lwsl_debug("%s: ctx %p freed\n", __func__, context); diff --git a/lib/core/private-lib-core.h b/lib/core/private-lib-core.h index 8bc0c9ffc..4ac1a46f5 100644 --- a/lib/core/private-lib-core.h +++ b/lib/core/private-lib-core.h @@ -281,6 +281,10 @@ struct lws; #include "private-lib-system-smd.h" #endif +#if defined(LWS_WITH_SYS_FAULT_INJECTION) +#include "private-lib-system-fault-injection.h" +#endif + struct lws_foreign_thread_pollfd { struct lws_foreign_thread_pollfd *next; int fd_index; @@ -442,6 +446,12 @@ struct lws_context { lws_async_dns_t async_dns; #endif +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + lws_fi_ctx_t fi; + /**< Toplevel Fault Injection ctx */ +#endif + + #if defined(LWS_WITH_SYS_NTPCLIENT) void *ntpclient_priv; #endif diff --git a/lib/secure-streams/private-lib-secure-streams.h b/lib/secure-streams/private-lib-secure-streams.h index 2109909c5..91c3705a2 100644 --- a/lib/secure-streams/private-lib-secure-streams.h +++ b/lib/secure-streams/private-lib-secure-streams.h @@ -54,6 +54,9 @@ typedef struct lws_ss_handle { #if defined(LWS_WITH_SERVER) struct lws_dll2 cli_list; /**< same server clients list */ #endif +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + lws_fi_ctx_t fi; /**< Fault Injection context */ +#endif struct lws_dll2_owner src_list; /**< sink's list of bound sources */ @@ -277,6 +280,10 @@ typedef struct lws_sspc_handle { struct lws_ss_serialization_parser parser; +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + lws_fi_ctx_t fi; /**< Fault Injection context */ +#endif + lws_dll2_owner_t metadata_owner; lws_dll2_owner_t metadata_owner_rx; diff --git a/lib/secure-streams/secure-streams-client.c b/lib/secure-streams/secure-streams-client.c index eed5b2342..3e4315b45 100644 --- a/lib/secure-streams/secure-streams-client.c +++ b/lib/secure-streams/secure-streams-client.c @@ -486,6 +486,13 @@ lws_sspc_create(struct lws_context *context, int tsi, const lws_ss_info_t *ssi, __lws_lc_tag(&context->lcg[LWSLCG_SSP_CLIENT], &h->lc, ssi->streamtype); +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + h->fi.name = "sspc"; + h->fi.parent = &context->fi; + if (ssi->fi) + lws_fi_import(&h->fi, ssi->fi); +#endif + memcpy(&h->ssi, ssi, sizeof(*ssi)); ua = (uint8_t *)&h[1]; memset(ua, 0, ssi->user_alloc); @@ -567,6 +574,10 @@ lws_sspc_destroy(lws_sspc_handle_t **ph) h->ss_dangling_connected = 0; } +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + lws_fi_destroy(&h->fi); +#endif + lws_sul_cancel(&h->sul_retry); lws_dll2_remove(&h->client_list); diff --git a/lib/secure-streams/secure-streams.c b/lib/secure-streams/secure-streams.c index 35ce7d83b..2cfcbd515 100644 --- a/lib/secure-streams/secure-streams.c +++ b/lib/secure-streams/secure-streams.c @@ -789,6 +789,13 @@ lws_ss_create(struct lws_context *context, int tsi, const lws_ss_info_t *ssi, __lws_lc_tag(&context->lcg[LWSLCG_WSI_SS_CLIENT], &h->lc, "%s", ssi->streamtype ? ssi->streamtype : "nostreamtype"); +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + h->fi.name = "ss"; + h->fi.parent = &context->fi; + if (ssi->fi) + lws_fi_import(&h->fi, ssi->fi); +#endif + h->info = *ssi; h->policy = pol; h->context = context; @@ -1153,6 +1160,10 @@ lws_ss_destroy(lws_ss_handle_t **ppss) lws_vhost_destroy(v); #endif +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + lws_fi_destroy(&h->fi); +#endif + lws_sul_cancel(&h->sul_timeout); /* confirm no sul left scheduled in handle or user allocation object */ diff --git a/lib/system/CMakeLists.txt b/lib/system/CMakeLists.txt index 78cc24006..9ad9d7094 100644 --- a/lib/system/CMakeLists.txt +++ b/lib/system/CMakeLists.txt @@ -56,6 +56,12 @@ if (LWS_WITH_NETWORK) if (LWS_WITH_SYS_SMD) add_subdir_include_dirs(smd) endif() + + if (LWS_WITH_SYS_FAULT_INJECTION) + include_directories(./fault-injection) + list(APPEND SOURCES + system/fault-injection/fault-injection.c) + endif() endif() diff --git a/lib/system/fault-injection/fault-injection.c b/lib/system/fault-injection/fault-injection.c new file mode 100644 index 000000000..c833c6d6c --- /dev/null +++ b/lib/system/fault-injection/fault-injection.c @@ -0,0 +1,152 @@ +/* + * lws System Fault Injection + * + * Copyright (C) 2019 - 2021 Andy Green + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "private-lib-core.h" + +#include + + +static lws_fi_priv_t * +lws_fi_lookup(lws_fi_ctx_t *fic, const char *name) +{ + lws_start_foreach_dll(struct lws_dll2 *, p, fic->fi_owner.head) { + lws_fi_priv_t *pv = lws_container_of(p, lws_fi_priv_t, list); + + if (!strcmp(pv->fi.name, name)) + return pv; + + } lws_end_foreach_dll(p); + + return NULL; +} + +int +lws_fi(lws_fi_ctx_t *fic, const char *name) +{ + lws_fi_priv_t *pv = NULL; + + do { + pv = lws_fi_lookup(fic, name); + + if (pv) { + int n; + + switch (pv->fi.type) { + case LWSFI_ALWAYS: + goto inject; + + case LWSFI_DETERMINISTIC: + pv->fi.times++; + if (pv->fi.times >= pv->fi.pre) + if (pv->fi.times < pv->fi.pre + pv->fi.count) + goto inject; + return 0; + + case LWSFI_PROBABILISTIC: + pv->fi.times = (unsigned long)(pv->fi.times * 3) ^ + (unsigned long)lws_now_usecs(); + if ((uint16_t)pv->fi.times % 101 >= pv->fi.pre) + goto inject; + return 0; + + case LWSFI_PATTERN: + n = (int)(pv->fi.times % pv->fi.pre); + if (pv->fi.pattern[n >> 3] & (1 << (n & 7))) + goto inject; + + return 0; + + default: + return 0; + } + } + + fic = fic->parent; + } while (fic); + + return 0; + +inject: + lwsl_warn("%s: Injecting fault %s->%s\n", __func__, fic->name, + pv->fi.name); + + return 1; +} + +int +lws_fi_add(lws_fi_ctx_t *fic, const lws_fi_t *fi) +{ + lws_fi_priv_t *pv; + size_t n = strlen(fi->name); + + pv = lws_malloc(sizeof(*pv) + n + 1, __func__); + if (!pv) + return 1; + + lws_dll2_clear(&pv->list); + + memcpy(&pv->fi, fi, sizeof(*fi)); + pv->fi.name = (const char *)&pv[1]; + memcpy(&pv[1], fi->name, n + 1); + + lws_dll2_add_tail(&pv->list, &fic->fi_owner); + + return 0; +} + +void +lws_fi_remove(lws_fi_ctx_t *fic, const char *name) +{ + lws_fi_priv_t *pv = lws_fi_lookup(fic, name); + + if (!pv) + return; + + lws_dll2_remove(&pv->list); + lws_free(pv); +} + +void +lws_fi_import(lws_fi_ctx_t *fic_dest, const lws_fi_ctx_t *fic_src) +{ + lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1, fic_src->fi_owner.head) { + lws_fi_priv_t *pv = lws_container_of(p, lws_fi_priv_t, list); + + lws_dll2_remove(&pv->list); + lws_dll2_add_tail(&pv->list, &fic_dest->fi_owner); + + } lws_end_foreach_dll_safe(p, p1); +} + +void +lws_fi_destroy(lws_fi_ctx_t *fic) +{ + lws_start_foreach_dll_safe(struct lws_dll2 *, p, p1, fic->fi_owner.head) { + lws_fi_priv_t *pv = lws_container_of(p, lws_fi_priv_t, list); + + lws_dll2_remove(&pv->list); + lws_free(pv); + + } lws_end_foreach_dll_safe(p, p1); +} diff --git a/lib/system/fault-injection/private-lib-system-fault-injection.h b/lib/system/fault-injection/private-lib-system-fault-injection.h new file mode 100644 index 000000000..ea01dc83a --- /dev/null +++ b/lib/system/fault-injection/private-lib-system-fault-injection.h @@ -0,0 +1,35 @@ +/* + * lws System Message Distribution + * + * Copyright (C) 2019 - 2021 Andy Green + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#if defined(LWS_WITH_SYS_FAULT_INJECTION) + +typedef struct lws_fi_priv { + lws_dll2_t list; + lws_fi_t fi; +} lws_fi_priv_t; + +void +lws_fi_import(lws_fi_ctx_t *fic_dest, const lws_fi_ctx_t *fic_src); + +#endif diff --git a/minimal-examples/http-client/minimal-http-client-fi/CMakeLists.txt b/minimal-examples/http-client/minimal-http-client-fi/CMakeLists.txt new file mode 100644 index 000000000..0e0d95792 --- /dev/null +++ b/minimal-examples/http-client/minimal-http-client-fi/CMakeLists.txt @@ -0,0 +1,37 @@ +project(lws-minimal-http-client-fi C) +cmake_minimum_required(VERSION 2.8.12) +find_package(libwebsockets CONFIG REQUIRED) +list(APPEND CMAKE_MODULE_PATH ${LWS_CMAKE_DIR}) +include(CheckCSourceCompiles) +include(LwsCheckRequirements) + +set(SAMP lws-minimal-http-client-fi) +set(SRCS minimal-http-client.c) + +set(requirements 1) +require_lws_config(LWS_ROLE_H1 1 requirements) +require_lws_config(LWS_WITH_CLIENT 1 requirements) +require_lws_config(LWS_WITH_SYS_STATE 1 requirements) +require_lws_config(LWS_WITH_SYS_FAULT_INJECTION 1 requirements) + +if (requirements) + add_executable(${SAMP} ${SRCS}) + + # if (LWS_CTEST_INTERNET_AVAILABLE) + # add_test(NAME http-client-warmcat COMMAND lws-minimal-http-client ) + # add_test(NAME http-client-warmcat-h1 COMMAND lws-minimal-http-client --h1) + # set_tests_properties(http-client-warmcat + # http-client-warmcat-h1 + # PROPERTIES + # WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/minimal-examples/http-client/minimal-http-client + # TIMEOUT 20) + # + #endif() + + if (websockets_shared) + target_link_libraries(${SAMP} websockets_shared ${LIBWEBSOCKETS_DEP_LIBS}) + add_dependencies(${SAMP} websockets_shared) + else() + target_link_libraries(${SAMP} websockets ${LIBWEBSOCKETS_DEP_LIBS}) + endif() +endif() diff --git a/minimal-examples/http-client/minimal-http-client-fi/README.md b/minimal-examples/http-client/minimal-http-client-fi/README.md new file mode 100644 index 000000000..9387f8c75 --- /dev/null +++ b/minimal-examples/http-client/minimal-http-client-fi/README.md @@ -0,0 +1,76 @@ +# lws minimal http client + +The application goes to either https://warmcat.com or +https://localhost:7681 (with `-l` option) and receives the page data. + +## build + +``` + $ cmake . && make +``` + +## usage + +Commandline option|Meaning +---|--- +-d |Debug verbosity in decimal, eg, -d15 +-l| Connect to https://localhost:7681 and accept selfsigned cert +--h1|Specify http/1.1 only using ALPN, rejects h2 even if server supports it +--server |set server name to connect to +-k|Apply tls option LCCSCF_ALLOW_INSECURE +-j|Apply tls option LCCSCF_ALLOW_SELFSIGNED +-m|Apply tls option LCCSCF_SKIP_SERVER_CERT_HOSTNAME_CHECK +-e|Apply tls option LCCSCF_ALLOW_EXPIRED +-v|Connection validity use 3s / 10s instead of default 5m / 5m10s +--nossl| disable ssl connection +--user | Set Basic Auth username +--password | Set Basic Auth password + +``` + $ ./lws-minimal-http-client +[2018/03/04 14:43:20:8562] USER: LWS minimal http client +[2018/03/04 14:43:20:8571] NOTICE: Creating Vhost 'default' port -1, 1 protocols, IPv6 on +[2018/03/04 14:43:20:8616] NOTICE: created client ssl context for default +[2018/03/04 14:43:20:8617] NOTICE: lws_client_connect_2: 0x1814dc0: address warmcat.com +[2018/03/04 14:43:21:1496] NOTICE: lws_client_connect_2: 0x1814dc0: address warmcat.com +[2018/03/04 14:43:22:0154] NOTICE: lws_client_interpret_server_handshake: incoming content length 26520 +[2018/03/04 14:43:22:0154] NOTICE: lws_client_interpret_server_handshake: client connection up +[2018/03/04 14:43:22:0169] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:0169] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:0169] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:0169] USER: RECEIVE_CLIENT_HTTP_READ: read 1015 +[2018/03/04 14:43:22:0174] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:0174] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:0174] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:0174] USER: RECEIVE_CLIENT_HTTP_READ: read 1015 +[2018/03/04 14:43:22:0179] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:0179] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:0179] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:0179] USER: RECEIVE_CLIENT_HTTP_READ: read 1015 +[2018/03/04 14:43:22:3010] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:3010] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:3010] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:3010] USER: RECEIVE_CLIENT_HTTP_READ: read 1015 +[2018/03/04 14:43:22:3015] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:3015] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:3015] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:3015] USER: RECEIVE_CLIENT_HTTP_READ: read 1015 +[2018/03/04 14:43:22:3020] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:3020] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:3020] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:3020] USER: RECEIVE_CLIENT_HTTP_READ: read 1015 +[2018/03/04 14:43:22:3022] USER: RECEIVE_CLIENT_HTTP_READ: read 1024 +[2018/03/04 14:43:22:3022] USER: RECEIVE_CLIENT_HTTP_READ: read 974 +[2018/03/04 14:43:22:3022] NOTICE: lws_http_client_read: transaction completed says -1 +[2018/03/04 14:43:23:3042] USER: Completed +``` + +You can also test the client Basic Auth support against the http-server/minimal-http-server-basicauth +example. In one console window run the server and in the other + +``` +$ lws-minimal-http-client -l --nossl --path /secret/index.html --user user --password password +``` + +The Basic Auth credentials for the test server are literally username "user" and password "password". + diff --git a/minimal-examples/http-client/minimal-http-client-fi/minimal-http-client.c b/minimal-examples/http-client/minimal-http-client-fi/minimal-http-client.c new file mode 100644 index 000000000..2027b83d4 --- /dev/null +++ b/minimal-examples/http-client/minimal-http-client-fi/minimal-http-client.c @@ -0,0 +1,346 @@ +/* + * lws-minimal-http-client + * + * Written in 2010-2021 by Andy Green + * + * This file is made available under the Creative Commons CC0 1.0 + * Universal Public Domain Dedication. + * + * This demonstrates the a minimal http client using lws. + * + * It visits https://warmcat.com/ and receives the html page there. You + * can dump the page data by changing the #if 0 below. + */ + +#include +#include +#include + +static int interrupted, bad = 1, status; +#if defined(LWS_WITH_HTTP2) +static int long_poll; +#endif +static struct lws *client_wsi; +static const char *ba_user, *ba_password; + +static const lws_retry_bo_t retry = { + .secs_since_valid_ping = 3, + .secs_since_valid_hangup = 10, +}; + +static int +callback_http(struct lws *wsi, enum lws_callback_reasons reason, + void *user, void *in, size_t len) +{ + switch (reason) { + + /* because we are protocols[0] ... */ + case LWS_CALLBACK_CLIENT_CONNECTION_ERROR: + lwsl_err("CLIENT_CONNECTION_ERROR: %s\n", + in ? (char *)in : "(null)"); + interrupted = 1; + break; + + case LWS_CALLBACK_ESTABLISHED_CLIENT_HTTP: + { + char buf[128]; + + lws_get_peer_simple(wsi, buf, sizeof(buf)); + status = (int)lws_http_client_http_response(wsi); + + lwsl_user("Connected to %s, http response: %d\n", + buf, status); + } +#if defined(LWS_WITH_HTTP2) + if (long_poll) { + lwsl_user("%s: Client entering long poll mode\n", __func__); + lws_h2_client_stream_long_poll_rxonly(wsi); + } +#endif + break; + +#if defined(LWS_WITH_HTTP_BASIC_AUTH) + + /* you only need this if you need to do Basic Auth */ + case LWS_CALLBACK_CLIENT_APPEND_HANDSHAKE_HEADER: + { + unsigned char **p = (unsigned char **)in, *end = (*p) + len; + char b[128]; + + if (!ba_user || !ba_password) + break; + + if (lws_http_basic_auth_gen(ba_user, ba_password, b, sizeof(b))) + break; + if (lws_add_http_header_by_token(wsi, WSI_TOKEN_HTTP_AUTHORIZATION, + (unsigned char *)b, (int)strlen(b), p, end)) + return -1; + + break; + } +#endif + + /* chunks of chunked content, with header removed */ + case LWS_CALLBACK_RECEIVE_CLIENT_HTTP_READ: + lwsl_user("RECEIVE_CLIENT_HTTP_READ: read %d\n", (int)len); +#if defined(LWS_WITH_HTTP2) + if (long_poll) { + char dotstar[128]; + lws_strnncpy(dotstar, (const char *)in, len, + sizeof(dotstar)); + lwsl_notice("long poll rx: %d '%s'\n", (int)len, + dotstar); + } +#endif +#if 0 /* enable to dump the html */ + { + const char *p = in; + + while (len--) + if (*p < 0x7f) + putchar(*p++); + else + putchar('.'); + } +#endif + return 0; /* don't passthru */ + + /* uninterpreted http content */ + case LWS_CALLBACK_RECEIVE_CLIENT_HTTP: + { + char buffer[1024 + LWS_PRE]; + char *px = buffer + LWS_PRE; + int lenx = sizeof(buffer) - LWS_PRE; + + if (lws_http_client_read(wsi, &px, &lenx) < 0) + return -1; + } + return 0; /* don't passthru */ + + case LWS_CALLBACK_COMPLETED_CLIENT_HTTP: + lwsl_user("LWS_CALLBACK_COMPLETED_CLIENT_HTTP\n"); + interrupted = 1; + bad = status != 200; + lws_cancel_service(lws_get_context(wsi)); /* abort poll wait */ + break; + + case LWS_CALLBACK_CLOSED_CLIENT_HTTP: + interrupted = 1; + bad = status != 200; + lws_cancel_service(lws_get_context(wsi)); /* abort poll wait */ + break; + + default: + break; + } + + return lws_callback_http_dummy(wsi, reason, user, in, len); +} + +static const struct lws_protocols protocols[] = { + { + "http", + callback_http, + 0, + 0, + }, + { NULL, NULL, 0, 0 } +}; + +static void +sigint_handler(int sig) +{ + interrupted = 1; +} + +struct args { + int argc; + const char **argv; +}; + +static const lws_fi_t fi = { + .name = "cliwsi.dnsfail", + .type = LWSFI_ALWAYS +}; + +static int +make_client_connection(struct lws_context *context) +{ + struct lws_client_connect_info i; + struct args *a = lws_context_user(context); + lws_fi_ctx_t fic; + const char *p; + + memset(&fic, 0, sizeof(fic)); + lws_fi_add(&fic, &fi); + + memset(&i, 0, sizeof i); /* otherwise uninitialized garbage */ + i.context = context; + i.fi = &fic; + + if (!lws_cmdline_option(a->argc, a->argv, "-n")) { + i.ssl_connection = LCCSCF_USE_SSL; +#if defined(LWS_WITH_HTTP2) + /* requires h2 */ + if (lws_cmdline_option(a->argc, a->argv, "--long-poll")) { + lwsl_user("%s: long poll mode\n", __func__); + long_poll = 1; + } +#endif + } + + if (lws_cmdline_option(a->argc, a->argv, "-l")) { + i.port = 7681; + i.address = "localhost"; + i.ssl_connection |= LCCSCF_ALLOW_SELFSIGNED; + } else { + i.port = 443; + i.address = "warmcat.com"; + } + + if (lws_cmdline_option(a->argc, a->argv, "--nossl")) + i.ssl_connection = 0; + + i.ssl_connection |= LCCSCF_H2_QUIRK_OVERFLOWS_TXCR | + LCCSCF_H2_QUIRK_NGHTTP2_END_STREAM; + + i.alpn = "h2"; + if (lws_cmdline_option(a->argc, a->argv, "--h1")) + i.alpn = "http/1.1"; + + if (lws_cmdline_option(a->argc, a->argv, "--h2-prior-knowledge")) + i.ssl_connection |= LCCSCF_H2_PRIOR_KNOWLEDGE; + + if ((p = lws_cmdline_option(a->argc, a->argv, "-p"))) + i.port = atoi(p); + + if ((p = lws_cmdline_option(a->argc, a->argv, "--user"))) + ba_user = p; + if ((p = lws_cmdline_option(a->argc, a->argv, "--password"))) + ba_password = p; + + if (lws_cmdline_option(a->argc, a->argv, "-j")) + i.ssl_connection |= LCCSCF_ALLOW_SELFSIGNED; + + if (lws_cmdline_option(a->argc, a->argv, "-k")) + i.ssl_connection |= LCCSCF_ALLOW_INSECURE; + + if (lws_cmdline_option(a->argc, a->argv, "-m")) + i.ssl_connection |= LCCSCF_SKIP_SERVER_CERT_HOSTNAME_CHECK; + + if (lws_cmdline_option(a->argc, a->argv, "-e")) + i.ssl_connection |= LCCSCF_ALLOW_EXPIRED; + + if ((p = lws_cmdline_option(a->argc, a->argv, "-f"))) { + i.ssl_connection |= LCCSCF_H2_MANUAL_RXFLOW; + i.manual_initial_tx_credit = atoi(p); + lwsl_notice("%s: manual peer tx credit %d\n", __func__, + i.manual_initial_tx_credit); + } + + /* the default validity check is 5m / 5m10s... -v = 3s / 10s */ + + if (lws_cmdline_option(a->argc, a->argv, "-v")) + i.retry_and_idle_policy = &retry; + + if ((p = lws_cmdline_option(a->argc, a->argv, "--server"))) + i.address = p; + + if ((p = lws_cmdline_option(a->argc, a->argv, "--path"))) + i.path = p; + else + i.path = "/"; + + i.host = i.address; + i.origin = i.address; + i.method = "GET"; + + i.protocol = protocols[0].name; + i.pwsi = &client_wsi; + + return !lws_client_connect_via_info(&i); +} + +static int +system_notify_cb(lws_state_manager_t *mgr, lws_state_notify_link_t *link, + int current, int target) +{ + struct lws_context *context = mgr->parent; + + if (current != LWS_SYSTATE_OPERATIONAL || target != LWS_SYSTATE_OPERATIONAL) + return 0; + + lwsl_info("%s: operational\n", __func__); + + make_client_connection(context); + + return 0; +} + +int main(int argc, const char **argv) +{ + lws_state_notify_link_t notifier = { {0}, system_notify_cb, "app" }; + lws_state_notify_link_t *na[] = { ¬ifier, NULL }; + struct lws_context_creation_info info; + struct lws_context *context; + struct args args; + int n = 0; + // uint8_t memcert[4096]; + + args.argc = argc; + args.argv = argv; + + signal(SIGINT, sigint_handler); + + memset(&info, 0, sizeof info); /* otherwise uninitialized garbage */ + lws_cmdline_option_handle_builtin(argc, argv, &info); + + lwsl_user("LWS minimal http client [-d] [-l] [--h1]\n"); + + info.options = LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT; + info.port = CONTEXT_PORT_NO_LISTEN; /* we do not run any server */ + info.protocols = protocols; + info.user = &args; + info.register_notifier_list = na; + info.connect_timeout_secs = 30; + + /* + * since we know this lws context is only ever going to be used with + * one client wsis / fds / sockets at a time, let lws know it doesn't + * have to use the default allocations for fd tables up to ulimit -n. + * It will just allocate for 1 internal and 1 (+ 1 http2 nwsi) that we + * will use. + */ + info.fd_limit_per_thread = 1 + 1 + 1; + +#if defined(LWS_WITH_MBEDTLS) || defined(USE_WOLFSSL) + /* + * OpenSSL uses the system trust store. mbedTLS has to be told which + * CA to trust explicitly. + */ + info.client_ssl_ca_filepath = "./warmcat.com.cer"; +#endif +#if 0 + n = open("./warmcat.com.cer", O_RDONLY); + if (n >= 0) { + info.client_ssl_ca_mem_len = read(n, memcert, sizeof(memcert)); + info.client_ssl_ca_mem = memcert; + close(n); + n = 0; + memcert[info.client_ssl_ca_mem_len++] = '\0'; + } +#endif + context = lws_create_context(&info); + if (!context) { + lwsl_err("lws init failed\n"); + return 1; + } + + while (n >= 0 && !interrupted) + n = lws_service(context, 0); + + lws_context_destroy(context); + lwsl_user("Completed: %s\n", bad ? "failed" : "OK"); + + return bad; +} diff --git a/minimal-examples/http-client/minimal-http-client-fi/warmcat.com.cer b/minimal-examples/http-client/minimal-http-client-fi/warmcat.com.cer new file mode 100644 index 000000000..8dc39efc5 --- /dev/null +++ b/minimal-examples/http-client/minimal-http-client-fi/warmcat.com.cer @@ -0,0 +1,21 @@ + -----BEGIN CERTIFICATE----- +MIIDSjCCAjKgAwIBAgIQRK+wgNajJ7qJMDmGLvhAazANBgkqhkiG9w0BAQUFADA/ +MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT +DkRTVCBSb290IENBIFgzMB4XDTAwMDkzMDIxMTIxOVoXDTIxMDkzMDE0MDExNVow +PzEkMCIGA1UEChMbRGlnaXRhbCBTaWduYXR1cmUgVHJ1c3QgQ28uMRcwFQYDVQQD +Ew5EU1QgUm9vdCBDQSBYMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +AN+v6ZdQCINXtMxiZfaQguzH0yxrMMpb7NnDfcdAwRgUi+DoM3ZJKuM/IUmTrE4O +rz5Iy2Xu/NMhD2XSKtkyj4zl93ewEnu1lcCJo6m67XMuegwGMoOifooUMM0RoOEq +OLl5CjH9UL2AZd+3UWODyOKIYepLYYHsUmu5ouJLGiifSKOeDNoJjj4XLh7dIN9b +xiqKqy69cK3FCxolkHRyxXtqqzTWMIn/5WgTe1QLyNau7Fqckh49ZLOMxt+/yUFw +7BZy1SbsOFU5Q9D8/RhcQPGX69Wam40dutolucbY38EVAjqr2m7xPi71XAicPNaD +aeQQmxkqtilX4+U9m5/wAl0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNV +HQ8BAf8EBAMCAQYwHQYDVR0OBBYEFMSnsaR7LHH62+FLkHX/xBVghYkQMA0GCSqG +SIb3DQEBBQUAA4IBAQCjGiybFwBcqR7uKGY3Or+Dxz9LwwmglSBd49lZRNI+DT69 +ikugdB/OEIKcdBodfpga3csTS7MgROSR6cz8faXbauX+5v3gTt23ADq1cEmv8uXr +AvHRAosZy5Q6XkjEGB5YGV8eAlrwDPGxrancWYaLbumR9YbK+rlmM6pZW87ipxZz +R8srzJmwN0jP41ZL9c8PDHIyh8bwRLtTcm1D9SZImlJnt1ir/md2cXjbDaJWFBM5 +JDGFoqgCWjBH4d1QB7wCCZAA62RjYJsWvIjJEubSfZGL+T0yjWW06XyxV3bqxbYo +Ob8VZRzI9neWagqNdwvYkQsEjgfbKbYK7p2CNTUQ +-----END CERTIFICATE----- +