lws_metrics

There are a few build options that are trying to keep and report
various statistics

 - DETAILED_LATENCY
 - SERVER_STATUS
 - WITH_STATS

remove all those and establish a generic rplacement, lws_metrics.

lws_metrics makes its stats available via an lws_system ops function
pointer that the user code can set.

Openmetrics export is supported, for, eg, prometheus scraping.

.sai.json

@@ -54,7 +54,7 @@
 			"build": "mkdir build destdir; cd build; export LD_LIBRARY_PATH=../destdir/usr/local/share/libwebsockets-test-server/plugins:../destdir/usr/local/lib;export SAI_CPACK=\"-G ZIP\";export MACOSX_DEPLOYMENT_TARGET=10.15 ; cmake .. -DCMAKE_MAKE_PROGRAM=/usr/bin/make -DLWS_OPENSSL_INCLUDE_DIRS=/usr/local/opt/openssl@1.1/include -DLWS_OPENSSL_LIBRARIES=\"/usr/local/opt/openssl/lib/libssl.dylib;/usr/local/opt/openssl/lib/libcrypto.dylib\" ${cmake} && make -j4 && make -j DESTDIR=../destdir install && ctest -j2 --output-on-failure ${cpack}"
 		},
 		"netbsd-OSX-bigsur/aarch64-apple-m1/llvm": {
-			"build": "mkdir build destdir; cd build; export LD_LIBRARY_PATH=../destdir/usr/local/share/libwebsockets-test-server/plugins:../destdir/usr/local/lib;export SAI_CPACK=\"-G ZIP\";export MACOSX_DEPLOYMENT_TARGET=10.15 ; cmake .. -DLWS_WITH_SUL_DEBUGGING=1 -DCMAKE_SYSTEM_PREFIX_PATH=/opt/homebrew -DLWS_OPENSSL_INCLUDE_DIRS=/opt/homebrew/Cellar/openssl@1.1/1.1.1h/include '-DLWS_OPENSSL_LIBRARIES=/opt/homebrew/Cellar/openssl@1.1/1.1.1h/lib/libssl.dylib;/opt/homebrew/Cellar/openssl@1.1/1.1.1h/lib/libcrypto.dylib' -DLWS_WITH_MINIMAL_EXAMPLES=1 ${cmake} && make -j6 && rm -rf ../destdir && make -j DESTDIR=../destdir install && ctest -j3 --output-on-failure ${cpack}"
+			"build": "mkdir build destdir; cd build; export LD_LIBRARY_PATH=../destdir/usr/local/share/libwebsockets-test-server/plugins:../destdir/usr/local/lib;export SAI_CPACK=\"-G ZIP\";export MACOSX_DEPLOYMENT_TARGET=10.15 ; cmake .. -DLWS_WITH_SUL_DEBUGGING=1 -DCMAKE_SYSTEM_PREFIX_PATH=/opt/homebrew -DLWS_OPENSSL_INCLUDE_DIRS=/opt/homebrew/Cellar/openssl@1.1/1.1.1h/include '-DLWS_OPENSSL_LIBRARIES=/opt/homebrew/Cellar/openssl@1.1/1.1.1h/lib/libssl.dylib;/opt/homebrew/Cellar/openssl@1.1/1.1.1h/lib/libcrypto.dylib' ${cmake} && make -j6 && rm -rf ../destdir && make -j DESTDIR=../destdir install && ctest -j3 --output-on-failure ${cpack}"
 		},
 		"solaris/x86_64-amd/gcc": {
 			"build": "mkdir build destdir; cd build; export SAI_CPACK=\"-G ZIP\";cmake .. ${cmake} && make -j 4 && make install DESTDIR=../destdir && ctest -j2 --output-on-failure ${cpack}",
@@ -115,7 +115,7 @@
 		},
 		"default-noudp": {
 			"cmake":	"-DLWS_WITH_UDP=0",
-			"platforms":	"w10/x86_64-amd/msvc, w10/x86_64-amd/noptmsvc, freertos-linkit/arm32-m4-mt7697-usi/gcc, linux-ubuntu-2004/aarch64-a72-bcm2711-rpi4/gcc, w10/x86_64-amd/mingw32, w10/x86_64-amd/mingw64, netbsd/aarch64BE-bcm2837-a53/gcc, w10/x86_64-amd/wmbedtls-msvc"
+			"platforms":	"w10/x86_64-amd/msvc, w10/x86_64-amd/noptmsvc, freertos-linkit/arm32-m4-mt7697-usi/gcc, linux-ubuntu-2004/aarch64-a72-bcm2711-rpi4/gcc, w10/x86_64-amd/mingw32, w10/x86_64-amd/mingw64, netbsd/aarch64BE-bcm2837-a53/gcc, w10/x86_64-amd/wmbedtlsmsvc"
 		},
 		"esp32-heltec": {
 			"cmake":	"",
@@ -181,7 +181,10 @@
 			"cmake":	"-DLWS_WITH_SECURE_STREAMS=1 -DLWS_WITH_SECURE_STREAMS_PROXY_API=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DLWS_WITH_SECURE_STREAMS_AUTH_SIGV4=1",
 			"platforms":	"not w10/x86_64-amd/msvc, netbsd/aarch64BE-bcm2837-a53/gcc, openbsd/x86_64-amd/llvm, solaris/x86_64-amd/gcc"
 		},
-
+		"secure-streams-proxy-metrics": {
+			"cmake":	"-DLWS_WITH_SECURE_STREAMS=1 -DLWS_WITH_SECURE_STREAMS_PROXY_API=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DLWS_WITH_SECURE_STREAMS_AUTH_SIGV4=1 -DLWS_WITH_SYS_METRICS=1",
+			"platforms":	"not w10/x86_64-amd/msvc, netbsd/aarch64BE-bcm2837-a53/gcc"
+		},
 		"distro_recommended": { # minimal examples also needed for ctest
 			"cmake":	"-DLWS_WITH_DISTRO_RECOMMENDED=1 -DLWS_WITH_MINIMAL_EXAMPLES=1",
 			"platforms":	"not freebsd-12/x86_64-amd/llvm, not linkit-cross, not w10/x86_64-amd/msvc, linux-ubuntu-2004/aarch64-a72-bcm2711-rpi4/gcc, linux-fedora-32/riscv64-virt/gcc",
@@ -193,6 +196,11 @@
 			# no distro -devel package for libuv
 			"platforms":	"not linux-centos-8/x86_64-amd/gcc"
 		},
+		"lwsws-nometrics": {
+			"cmake":	"-DLWS_WITH_LWSWS=ON -DLWS_WITHOUT_EXTENSIONS=0 -DLWS_WITH_HTTP2=1 -DLWS_WITH_ACME=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DCMAKE_BUILD_TYPE=DEBUG -DLWS_WITH_GENCRYPTO=1 -DLWS_WITH_JOSE=1 -DLWS_WITH_SYS_ASYNC_DNS=1 -DLWS_WITH_SYS_NTPCLIENT=1 -DLWS_WITH_SYS_METRICS=0",
+			# no distro -devel package for libuv
+			"platforms":	"not linux-centos-8/x86_64-amd/gcc"
+		},
 		"lwsws2": {
 			"cmake":	"-DLWS_WITH_LWSWS=ON -DLWS_WITHOUT_EXTENSIONS=0 -DLWS_WITH_HTTP2=1 -DLWS_WITH_ACME=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DCMAKE_BUILD_TYPE=DEBUG -DLWS_WITH_LWS_DSH=1",
 			# no distro -devel package for libuv
@@ -207,6 +215,10 @@
 			# no distro -devel package for mbedtls
 			"platforms":	"not linux-centos-7/x86_64-amd/gcc, not linux-centos-8/x86_64-amd/gcc"
 		},
+		"mbedtls-metrics": {
+			"cmake":	"-DLWS_WITH_MBEDTLS=1 -DLWS_WITH_HTTP2=1 -DLWS_WITH_LWSWS=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DLWS_WITH_JOSE=1 -DCMAKE_BUILD_TYPE=DEBUG -DLWS_WITH_SYS_METRICS=1",
+			"platforms":	"not linux-centos-7/x86_64-amd/gcc, not linux-centos-8/x86_64-amd/gcc"
+		},
 		"noserver": {
 			"cmake":	"-DLWS_WITHOUT_SERVER=ON -DLWS_WITH_MINIMAL_EXAMPLES=1 -DLWS_WITH_SECURE_STREAMS=1",
 			"platforms":    "w10/x86_64-amd/msvc, w10/x86_64-amd/noptmsvc"

CMakeLists-implied-options.txt

@@ -83,7 +83,7 @@ if(LWS_WITH_DISTRO_RECOMMENDED)
 	set(LWS_WITH_SOCKS5 1)				# selfcontained
 	set(LWS_WITH_RANGES 1)				# selfcontained
 	set(LWS_WITH_ACME 1)				# selfcontained / tls
-	set(LWS_WITH_SERVER_STATUS 1)			# selfcontained
+	set(LWS_WITH_SYS_METRICS 1)			# selfcontained
 	set(LWS_WITH_GLIB 1)				# glib
 	set(LWS_WITH_LIBUV 1)				# libuv
 	set(LWS_WITH_LIBEV 1)				# libev
@@ -128,6 +128,7 @@ endif()
 if (LWS_WITH_SECURE_STREAMS_PROXY_API)
 	set(LWS_WITH_LWS_DSH 1)
 	set(LWS_WITH_UNIX_SOCK 1)
+	set(LWS_WITH_SYS_SMD 1)
 endif()
 
 if (NOT LWS_WITH_NETWORK)
@@ -210,7 +211,7 @@ if (LWS_WITH_LWSWS)
  set(LWS_WITH_LIBUV_INTERNAL 1)
  set(LWS_WITH_EVENT_LIBS 1) # implied by LIBUV_INTERNAL
  set(LWS_WITH_ACCESS_LOG 1)
- set(LWS_WITH_SERVER_STATUS 1)
+ set(LWS_WITH_SYS_METRICS 1)
  set(LWS_WITH_LEJP 1)
  set(LWS_WITH_LEJP_CONF 1)
  set(LWS_WITH_PEER_LIMITS 1)

CMakeLists.txt

@@ -112,7 +112,6 @@ option(LWS_WITH_SOCKS5 "Allow use of SOCKS5 proxy on client connections" OFF)
 option(LWS_WITH_PEER_LIMITS "Track peers and restrict resources a single peer can allocate" OFF)
 option(LWS_WITH_ACCESS_LOG "Support generating Apache-compatible access logs" OFF)
 option(LWS_WITH_RANGES "Support http ranges (RFC7233)" OFF)
-option(LWS_WITH_SERVER_STATUS "Support json + jscript server monitoring" OFF)
 option(LWS_WITH_THREADPOOL "Managed worker thread pool support (relies on pthreads)" OFF)
 option(LWS_WITH_HTTP_STREAM_COMPRESSION "Support HTTP stream compression" OFF)
 option(LWS_WITH_HTTP_BROTLI "Also offer brotli http stream compression (requires LWS_WITH_HTTP_STREAM_COMPRESSION)" OFF)
@@ -134,6 +133,7 @@ else()
 	option(LWS_WITH_RFC6724 "Enable RFC6724 DNS result sorting" OFF)
 endif()
 option(LWS_WITH_SYS_FAULT_INJECTION "Enable fault injection support" OFF)
+option(LWS_WITH_SYS_METRICS "Lws Metrics API" OFF)
 
 #
 # Secure Streams
@@ -250,7 +250,6 @@ set(LWS_LOGGING_BITFIELD_CLEAR 0 CACHE STRING "Bitfield describing which log lev
 option(LWS_LOGS_TIMESTAMP "Timestamp at start of logs" ON)
 option(LWS_LOG_TAG_LIFECYCLE "Log tagged object lifecycle as NOTICE" ON)
 option(LWS_AVOID_SIGPIPE_IGN "Android 7+ reportedly needs this" OFF)
-option(LWS_WITH_STATS "Keep statistics of lws internal operations" OFF)
 option(LWS_WITH_JOSE "JSON Web Signature / Encryption / Keys (RFC7515/6/) API" OFF)
 option(LWS_WITH_GENCRYPTO "Enable support for Generic Crypto apis independent of TLS backend" OFF)
 option(LWS_WITH_SELFTESTS "Selftests run at context creation" OFF)
@@ -272,7 +271,6 @@ option(LWS_WITH_EXTERNAL_POLL "Support external POLL integration using callback
 option(LWS_WITH_LWS_DSH "Support lws_dsh_t Disordered Shared Heap" OFF)
 option(LWS_CLIENT_HTTP_PROXYING "Support external http proxies for client connections" ON)
 option(LWS_WITH_FILE_OPS "Support file operations vfs" ON)
-option(LWS_WITH_DETAILED_LATENCY "Record detailed latency stats for each read and write" OFF)
 option(LWS_WITH_UDP "Platform supports UDP" ON)
 option(LWS_WITH_SPAWN "Spawn subprocesses with piped stdin/out/stderr" OFF)
 option(LWS_WITH_FSMOUNT "Overlayfs and fallback mounting apis" OFF)

README.md

@@ -378,7 +378,8 @@ with `api-tests/api-test-async-dns` minimal example.
 You can now opt to measure and store us-resolution statistics on effective
 latencies for client operations, and easily spool them to a file in a
 format suitable for gnuplot, or handle in your own callback.  Enable
-`-DLWS_WITH_DETAILED_LATENCY=1` in cmake to build it into lws.
+`-DLWS_WITH_DETAILED_LATENCY=1` in cmake to build it into lws. (NB 2021-01-12
+this has been replaced by the lws_metrics support)
 
 If you are concerned about operation latency or potential blocking from
 user code, or behaviour under load, or latency variability on specific

READMEs/README.detailed-latency.md

@@ -1,117 +0,0 @@
-# lws detailed latency
-
-![lws detailed latency example plot](../doc-assets/lws-detailed-latency-example.png)
-
-## Introduction
-
-lws has the capability to make detailed latency measurements and
-report them in realtime to a specified callback.
-
-A default callback is provided that renders the data as text in
-space-separated format suitable for gnuplot, to a specified file.
-
-## Configuring
-
-Enable `LWS_WITH_DETAILED_LATENCY` at cmake.
-
-Create your context with something similar to this
-
-```
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	info.detailed_latency_cb = lws_det_lat_plot_cb;
-	info.detailed_latency_filepath = "/tmp/lws-latency-results";
-#endif
-```
-
-`lws_det_lat_plot_cb` is provided by lws as a convenience to convert
-the stuct data provided at the callback interface to space-separated
-text data that is easy to process with shell commands and gnuplot.
-
-## `lws_det_lat_plot_cb` format
-
-```
-728239173547 N 23062 0 0 23062 0 0 0
-728239192554 C 18879 0 0 18879 0 0 0
-728239217894 T 25309 0 0 25309 0 0 0
-728239234998 r 0 0 0 0 271 172 256
-728239250611 r 0 0 0 0 69 934 4096
-728239255679 w 19 122 18 159 20 80 80
-728239275718 w 20 117 15 152 18 80 80
-728239295578 w 10 73 7 90 7 80 80
-728239315567 w 9 67 5 81 7 80 80
-728239335745 w 23 133 9 165 14 80 80
-...
-```
-
-Each event is shown in 9 columns
-
- - unix time in us
- - event type
-   - N = Name resolution
-   - C = TCP Connection
-   - T = TLS negotiation server
-   - t = TLS negotiation client
-   - r = Read
-   - w = Write
- - us duration, for w time client spent waiting to write
- - us duration, for w time data spent in transit to proxy
- - us duration, for w time proxy waited to send data
- - as a convenience, sum of last 3 columns above
- - us duration, time spent in callback
- - last 2 are actual / requested size in bytes
-
-## Processing captured data with ministat
-
-Eg, to summarize overall latencies on all captured writes
-
-```
- $ cat /tmp/lws-latency-results | grep " w " | cut -d' ' -f6 | ministat
-...
-    N           Min           Max        Median           Avg        Stddev
-x 1000            43           273           141       132.672     32.471693
-```
-
-## Processing captured data with gnuplot
-
-### Gnuplot plotting script
-
-Create a gnuplot script, eg myscript.gp
-
-```
-reset
-set term pngcairo enhanced nocrop font "OpenSans, 12" size 800,600#output terminal and file
-set output "lws-latency.png"
-#set yrange [0:10000]
-#to put an empty boundary around the
-#data inside an autoscaled graph.
-set offset graph 0.05,0.05,0.05,0.0
-set style fill transparent solid 0.5 #fillstyle
-set tics out nomirror
-set xlabel "event"
-set ylabel "latency (us)"
-set format x ""
-set title "Write latency"
-set key invert reverse Right inside nobox
-set key autotitle columnheader
-set style data histogram
-set style histogram rowstacked
-set style fill solid border -1
-set boxwidth 0.75
-set style fill solid 1.00 noborder
-set tic scale 0
-set grid ytics lc rgb "#505050"
-unset border
-unset xtics
-
-plot '/tmp/1' \
-	   using ($3 + $4 + $5):xtic(1)         w boxes lt rgbcolor "blue"  title 'prox wr wait', \
-	'' using ($3 + $4):xtic(1)         w boxes lt rgbcolor "green" title 'txfr to prox', \
-	'' using 3:xtic(1) w boxes lt rgbcolor "red"   title 'cli wri wait'
-```
-
-### gnuplot invocation
-
-```
- $ cat /tmp/lws-latency-results | grep " w " \>/tmp/1 ; gnuplot myscript.gp && eog lws-latency.png
-```
-

READMEs/README.lws_metrics.md

@@ -0,0 +1,245 @@
+## `lws_metrics`
+
+### Introduction
+
+`lws_metrics` records and aggregates **events** at all lws layers.
+
+There are three distinct parts:
+
+ - the architecture inside lws for collecting and aggregating / decimating the
+   events and maintaining statistics about them, these are lws_metric objects
+
+ - an external handler for forwarding aggregated metrics.  An lws_system ops
+   interface to pass on the aggregated metrics to an external backend.  lws
+   presents its own public metrics objects and leaves it to the external
+   code to have a shim to marry the lws metrics up to whatever is needed in the
+   metrics backend
+
+ - a policy for when to emit each type of aggregated information to the external
+   handler.  This can be specified in the generic Secure Streams policy, or
+   a linked-list of lws_metric_policy_t object passed it at context creation in
+   `info.metrics_policies`.
+
+The external backend interface code may itself make use of lws connectivity apis
+including Secure Streams itself, and lws metrics are available on that too.
+
+### `lws_metrics` policy-based reporting
+
+Normally metrics implementations are fixed at build-time and cannot change
+without a coordinated reflash of devices along with a change of backend schema.
+
+`lws_metrics` separates out the objects and code necessary to collect and
+aggregate the data cheaply, and the reporting policy that controls if, or how
+often, the results are reported to the external handler.
+
+![policy based metrics](/doc-assets/lws_metrics-policy.png)
+
+Metrics are created with a namespace name and the policy applies itself to those
+by listing the names, with wildcards allowed, the policy applies to, eg if
+specified in the Secure Streams JSON policy
+
+```
+	...
+	"metrics": [
+                {
+                        "name":         "tensecs",
+                        "us_schedule":  10000000,
+                        "report":	"cpu.*"
+                }, {
+                        "name":         "30secs",
+                        "us_schedule":  30000000,
+                        "report":       "n.cn.*, n.http.*, n.ss.*, vh.*"
+                }
+        ],
+        ...
+```
+
+Metrics that do not have a reporting policy do not report, but continue to
+aggregate measurements in case they are bound to a policy dynamically later.
+
+### Freeform metrics naming
+
+There is no predefined metrics schema, metrics objects, including those created
+by applications, can independently choose their own name in a namespace like
+"cpu.srv" or "n.cn.dns", and can set a prefix for all metrics names created in a
+context (by setting `info.metrics_prefix` at context creation time).
+
+This allows multiple processes in a single device to expose copies of the same
+metrics in an individually addressable way, eg, if the UI process specifies the
+prefix "ui", then its lws metrics like "cpu.srv" will actually be created as
+"ui.cpu.srv".
+
+Applications can freely define their own `lws_metrics` measurements with their
+own names in the namespace too, without central registration, and refer to those
+names in the reporting policy same as any other metric names.
+
+If the metrics backend requires a fixed schema, the mapping between the
+`lws_metrics` names and the backend schema indexes will be done in the
+`lws_system` external reporting api implementation alone.  Metrics objects
+contain a `void * backend_opaque` that is ignored by lws and can be set and
+read by the external reporting handler implementation to facilitate that.
+
+### Histogram metrics tagging
+
+Histogram metrics track differently-qualified results in the same metric, for
+example the metric `n.cn.failures` maintains separate result counts for all
+variations and kinds of failure.
+
+```
+[2021/03/01 06:34:05:6570] U: my_metric_report: ssproxy.n.cn.failures{ss="badcert_selfsigned",hostname="invalidca.badcert.warmcat.com",peer="46.105.127.147",tls="invalidca"} 2
+[2021/03/01 06:34:05:6573] U: my_metric_report: ssproxy.n.cn.failures{hostname="invalidca.badcert.warmcat.com",peer="46.105.127.147",tls="invalidca"} 1
+[2021/03/01 06:34:05:6576] U: my_metric_report: ssproxy.n.cn.failures{ss="badcert_expired",hostname="warmcat.com",peer="46.105.127.147",tls="expired"} 2
+[2021/03/01 06:34:05:6578] U: my_metric_report: ssproxy.n.cn.failures{hostname="warmcat.com",peer="46.105.127.147",tls="expired"} 1
+[2021/03/01 06:34:05:6580] U: my_metric_report: ssproxy.n.cn.failures{ss="badcert_hostname",hostname="hostname.badcert.warmcat.com",peer="46.105.127.147",tls="hostname"} 2
+[2021/03/01 06:34:05:6583] U: my_metric_report: ssproxy.n.cn.failures{hostname="hostname.badcert.warmcat.com",peer="46.105.127.147",tls="hostname"} 1
+[2021/03/01 06:34:05:6585] U: my_metric_report: ssproxy.n.cn.failures{dns="nores -2"} 8
+```
+
+The user handler for metrics is expected to iterate these, in the provided
+examples (eg, minimal-secure-streams-testsfail)
+
+```
+#if defined(LWS_WITH_SYS_METRICS)
+static int
+my_metric_report(lws_metric_pub_t *mp)
+{
+	lws_metric_bucket_t *sub = mp->u.hist.head;
+	char buf[192];
+
+	do {
+		if (lws_metrics_format(mp, &sub, buf, sizeof(buf)))
+			lwsl_user("%s: %s\n", __func__, buf);
+	} while ((mp->flags & LWSMTFL_REPORT_HIST) && sub);
+
+	/* 0 = leave metric to accumulate, 1 = reset the metric */
+
+	return 1;
+}
+
+static const lws_system_ops_t system_ops = {
+	.metric_report = my_metric_report,
+};
+
+#endif
+```
+
+### `lws_metrics` decimation
+
+Event information can easily be produced faster than it can be transmitted, or
+is useful to record if everything is working.  In the case that things are not
+working, then eventually the number of events that are unable to be forwarded
+to the backend would overwhelm the local storage.
+
+For that reason, the metrics objects are designed to absorb and summarize a
+potentially large number of events cheaply by aggregating them, so even extreme
+situations can be tracked meaningfully inbetween dumps to the backend.
+
+There are two approaches:
+
+ - "aggregation": decimate keeping a uint64 mean + sum, along with a max and min
+ 
+ - "histogram": keep a linked-list of different named buckets, with a 64-bit
+   counter for the number of times an event in each bucket was observed
+
+A single metric aggregation object has separate "go / no-go" counters, since
+most operations can fail, and failing operations act differently.
+
+`lws_metrics` 'aggregation' supports decimation by
+
+ - a mean of a 64-bit event metric, separate for go and no-go events
+ - counters of go and no-go events
+ - a min and max of the metric
+ - keeping track of when the sample period started
+
+![metrics decimation](/doc-assets/lws_metrics-decimation.png)
+
+In addition, the policy defines a percentage variance from the mean that
+optionally qualifies events to be reported individually.
+
+The `lws_metrics` 'histogram' allows monitoring of different outcomes to
+produce counts of each outcome in the "bucket".  
+
+### `lws_metrics` flags
+
+When the metrics object is created, flags are used to control how it will be
+used and consumed.
+
+For example to create a histogram metrics object rather than the default
+aggregation type, you would give the flag `LWSMTFL_REPORT_HIST` at creation
+time.
+
+|Flag|Meaning|
+|---|---|
+|`LWSMTFL_REPORT_OUTLIERS`|track outliers and report them internally|
+|`LWSMTFL_REPORT_OUTLIERS_OOB`|report each outlier externally as they happen|
+|`LWSMTFL_REPORT_INACTIVITY_AT_PERIODIC`|explicitly externally report no activity at periodic cb, by default no events in the period is just not reported|
+|`LWSMTFL_REPORT_MEAN`|the mean is interesting for this metric|
+|`LWSMTFL_REPORT_ONLY_GO`|no-go pieces invalid and should be ignored, used for simple counters|
+|`LWSMTFL_REPORT_DUTY_WALLCLOCK_US`|the aggregated sum or mean can be compared to wallclock time| 
+|`LWSMTFL_REPORT_HIST`|object is a histogram (else aggregator)|
+
+### Built-in lws-layer metrics
+
+lws creates and maintains various well-known metrics when you enable build
+with cmake `-DLWS_WITH_SYS_METRICS=1`:
+
+#### Aggregation metrics
+|metric name|scope|type|meaning|
+---|---|---|---|
+`cpu.svc`|context|monotonic over time|time spent servicing, outside of event loop wait|
+`n.cn.dns`|context|go/no-go mean|duration of blocking libc DNS lookup|
+`n.cn.adns`|context|go/no-go mean|duration of SYS_ASYNC_DNS lws DNS lookup|
+`n.cn.tcp`|context|go/no-go mean|duration of tcp connection until accept|
+`n.cn.tls`|context|go/no-go mean|duration of tls connection until accept|
+`n.http.txn`|context|go (2xx)/no-go mean|duration of lws http transaction|
+`n.ss.conn`|context|go/no-go mean|duration of Secure Stream transaction|
+`n.ss.cliprox.conn`|context|go/no-go mean|time taken for client -> proxy connection|
+`vh.[vh-name].rx`|vhost|go/no-go sum|received data on the vhost|
+`vh.[vh-name].tx`|vhost|go/no-go sum|transmitted data on the vhost|
+
+#### Histogram metrics
+|metric name|scope|type|meaning|
+|---|---|---|---|
+`n.cn.failures`|context|histogram|Histogram of connection attempt failure reasons|
+
+#### Connection failure histogram buckets
+|Bucket name|Meaning|
+|---|---|
+`tls/invalidca`|Peer certificate CA signature missing or not trusted|
+`tls/hostname`|Peer certificate CN or SAN doesn't match the endpoint we asked for|
+`tls/notyetvalid`|Peer certificate start date is in the future (time wrong?)|
+`tls/expired`|Peer certificate is expiry date is in the past|
+`dns/badsrv`|No DNS result because couldn't talk to the server|
+`dns/nxdomain`|No DNS result because server says no result|
+
+The `lws-minimal-secure-streams` example is able to report the aggregated
+metrics at the end of execution, eg
+
+```
+[2021/01/13 11:47:19:9145] U: my_metric_report: cpu.svc: 137.045ms / 884.563ms (15%)
+[2021/01/13 11:47:19:9145] U: my_metric_report: n.cn.dns: Go: 4, mean: 3.792ms, min: 2.470ms, max: 5.426ms
+[2021/01/13 11:47:19:9145] U: my_metric_report: n.cn.tcp: Go: 4, mean: 40.633ms, min: 17.107ms, max: 94.560ms
+[2021/01/13 11:47:19:9145] U: my_metric_report: n.cn.tls: Go: 3, mean: 91.232ms, min: 30.322ms, max: 204.635ms
+[2021/01/13 11:47:19:9145] U: my_metric_report: n.http.txn: Go: 4, mean: 63.089ms, min: 20.184ms, max: 125.474ms
+[2021/01/13 11:47:19:9145] U: my_metric_report: n.ss.conn: Go: 4, mean: 161.740ms, min: 42.937ms, max: 429.510ms
+[2021/01/13 11:47:19:9145] U: my_metric_report: vh._ss_default.rx: Go: (1) 102, NoGo: (1) 0
+[2021/01/13 11:47:19:9145] U: my_metric_report: vh.le_via_dst.rx: Go: (22) 28.165Ki
+[2021/01/13 11:47:19:9145] U: my_metric_report: vh.le_via_dst.tx: Go: (1) 267
+[2021/01/13 11:47:19:9145] U: my_metric_report: vh.api_amazon_com.rx: Go: (1) 1.611Ki, NoGo: (1) 0
+[2021/01/13 11:47:19:9145] U: my_metric_report: vh.api_amazon_com.tx: Go: (3) 1.505Ki
+```
+
+lws-minimal-secure-stream-testsfail which tests various kinds of connection failure
+reports histogram results like this
+
+```
+[2021/01/15 13:10:16:0933] U: my_metric_report: n.cn.failures: tot: 36, [ tls/invalidca: 5, tls/expired: 5, tls/hostname: 5, dns/nxdomain: 21 ]
+```
+
+## Support for openmetrics
+
+Openmetrics https://tools.ietf.org/html/draft-richih-opsawg-openmetrics-00
+defines a textual metrics export format comaptible with Prometheus.  Lws
+provides a protocol plugin in `./plugins/protocol_lws_openmetrics_export`
+that enables direct export for prometheus scraping, and also protocols to
+proxy openmetrics export for unreachable servers.

cmake/lws_config.h.in

@@ -195,6 +195,7 @@
 #cmakedefine LWS_WITH_SQLITE3
 #cmakedefine LWS_WITH_SYS_DHCP_CLIENT
 #cmakedefine LWS_WITH_SYS_FAULT_INJECTION
+#cmakedefine LWS_WITH_SYS_METRICS
 #cmakedefine LWS_WITH_SYS_NTPCLIENT
 #cmakedefine LWS_WITH_SYS_STATE
 #cmakedefine LWS_WITH_THREADPOOL

include/libwebsockets.h

@@ -575,8 +575,8 @@ struct lws;
 #include <libwebsockets/lws-retry.h>
 #include <libwebsockets/lws-adopt.h>
 #include <libwebsockets/lws-network-helper.h>
+#include <libwebsockets/lws-metrics.h>
 #include <libwebsockets/lws-system.h>
-#include <libwebsockets/lws-detailed-latency.h>
 #include <libwebsockets/lws-ws-close.h>
 #include <libwebsockets/lws-callbacks.h>
 #include <libwebsockets/lws-ws-state.h>
@@ -605,7 +605,6 @@ struct lws;
 #include <libwebsockets/lws-vfs.h>
 #endif
 #include <libwebsockets/lws-lejp.h>
-#include <libwebsockets/lws-stats.h>
 #include <libwebsockets/lws-struct.h>
 #include <libwebsockets/lws-threadpool.h>
 #include <libwebsockets/lws-tokenize.h>

include/libwebsockets/lws-context-vhost.h

@@ -245,6 +245,7 @@
 struct lws_plat_file_ops;
 struct lws_ss_policy;
 struct lws_ss_plugin;
+struct lws_metric_policy;
 
 typedef int (*lws_context_ready_cb_t)(struct lws_context *context);
 
@@ -725,13 +726,6 @@ struct lws_context_creation_info {
 	const lws_system_ops_t *system_ops;
 	/**< CONTEXT: hook up lws_system_ apis to system-specific
 	 * implementations */
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	det_lat_buf_cb_t detailed_latency_cb;
-	/**< CONTEXT: NULL, or callback to receive detailed latency information
-	 * collected for each read and write */
-	const char *detailed_latency_filepath;
-	/**< CONTEXT: NULL, or filepath to put latency data into */
-#endif
 	const lws_retry_bo_t *retry_and_idle_policy;
 	/**< VHOST: optional retry and idle policy to apply to this vhost.
 	 *   Currently only the idle parts are applied to the connections.
@@ -840,6 +834,17 @@ struct lws_context_creation_info {
 	 * (20 for FREERTOS) */
 #endif
 
+#if defined(LWS_WITH_SYS_METRICS)
+	const struct lws_metric_policy		*metrics_policies;
+	/**< non-SS policy metrics policies */
+	const char				*metrics_prefix;
+	/**< prefix for this context's metrics, used to distinguish metrics
+	 * pooled from different processes / applications, so, eg what would
+	 * be "cpu.svc" if this is NULL becomes "myapp.cpu.svc" is this is
+	 * set to "myapp".  Policies are applied using the name with the prefix,
+	 * if present.
+	 */
+#endif
 
 	/* Add new things just above here ---^
 	 * This is part of the ABI, don't needlessly break compatibility

include/libwebsockets/lws-detailed-latency.h

@@ -1,140 +0,0 @@
-/*
- * libwebsockets - small server side websockets and web server implementation
- *
- * Copyright (C) 2010 - 2019 Andy Green <andy@warmcat.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- *
- * included from libwebsockets.h
- */
-
-enum {
-
-	/* types of latency, all nonblocking except name resolution */
-
-	LDLT_READ,	/* time taken to read LAT_DUR_PROXY_RX_TO_CLIENT_WRITE */
-	LDLT_WRITE,
-	LDLT_NAME_RESOLUTION, /* BLOCKING: LAT_DUR_PROXY_CLIENT_REQ_TO_WRITE */
-	LDLT_CONNECTION, /* conn duration: LAT_DUR_PROXY_CLIENT_REQ_TO_WRITE */
-	LDLT_TLS_NEG_CLIENT, /* tls conn duration: LAT_DUR_PROXY_CLIENT_REQ_TO_WRITE */
-	LDLT_TLS_NEG_SERVER, /* tls conn duration: LAT_DUR_PROXY_CLIENT_REQ_TO_WRITE */
-
-	LDLT_USER,
-
-	/* interval / duration elements in latencies array */
-
-	LAT_DUR_PROXY_CLIENT_REQ_TO_WRITE				= 0,
-		/* us the client spent waiting to write to proxy */
-	LAT_DUR_PROXY_CLIENT_WRITE_TO_PROXY_RX,
-		/* us the packet took to be received by proxy */
-	LAT_DUR_PROXY_PROXY_REQ_TO_WRITE,
-		/* us the proxy has to wait before it could write */
-	LAT_DUR_PROXY_RX_TO_ONWARD_TX,
-		/* us the proxy spent waiting to write to destination, or
-		 * if nonproxied, then time between write request and write */
-
-	LAT_DUR_USERCB, /* us duration of user callback */
-
-	LAT_DUR_STEPS /* last */
-};
-
-typedef struct lws_detlat {
-	lws_usec_t		earliest_write_req;
-	lws_usec_t		earliest_write_req_pre_write;
-		/**< use this for interval comparison */
-	const char		*aux; /* name for name resolution timing */
-	int			type;
-	uint32_t		latencies[LAT_DUR_STEPS];
-	size_t			req_size;
-	size_t			acc_size;
-} lws_detlat_t;
-
-typedef int (*det_lat_buf_cb_t)(struct lws_context *context,
-				const lws_detlat_t *d);
-
-/**
- * lws_det_lat_cb() - inject your own latency records
- *
- * \param context: the lws_context
- * \param d: the lws_detlat_t you have prepared
- *
- * For proxying or similar cases where latency information is available from
- * user code rather than lws itself, you can generate your own latency callback
- * events with your own lws_detlat_t.
- */
-
-LWS_VISIBLE LWS_EXTERN int
-lws_det_lat_cb(struct lws_context *context, lws_detlat_t *d);
-
-/*
- * detailed_latency_plot_cb() - canned save to file in plottable format cb
- *
- * \p context: the lws_context
- * \p d: the detailed latency event information
- *
- * This canned callback makes it easy to export the detailed latency information
- * to a file.  Just set the context creation members like this
- *
- * #if defined(LWS_WITH_DETAILED_LATENCY)
- *	info.detailed_latency_cb = lws_det_lat_plot_cb;
- *	info.detailed_latency_filepath = "/tmp/lws-latency-results";
- * #endif
- *
- * and you will get a file containing information like this
- *
- * 718823864615 N 10589 0 0 10589 0 0 0
- * 718823880837 C 16173 0 0 16173 0 0 0
- * 718823913063 T 32212 0 0 32212 0 0 0
- * 718823931835 r 0 0 0 0 232 30 256
- * 718823948757 r 0 0 0 0 40 30 256
- * 718823948799 r 0 0 0 0 83 30 256
- * 718823965602 r 0 0 0 0 27 30 256
- * 718823965617 r 0 0 0 0 43 30 256
- * 718823965998 r 0 0 0 0 12 28 256
- * 718823983887 r 0 0 0 0 74 3 4096
- * 718823986411 w 16 87 7 110 9 80 80
- * 718824006358 w 8 68 6 82 6 80 80
- *
- * which is easy to grep and pass to gnuplot.
- *
- * The columns are
- *
- *  - unix time in us
- *  - N = Name resolution, C = TCP Connection, T = TLS negotiation server,
- *    t = TLS negotiation client, r = Read, w = Write
- *  - us duration, for w time client spent waiting to write
- *  - us duration, for w time data spent in transit to proxy
- *  - us duration, for w time proxy waited to send data
- *  - as a convenience, sum of last 3 columns above
- *  - us duration, time spent in callback
- *  - last 2 are actual / requested size in bytes
- */
-LWS_VISIBLE LWS_EXTERN int
-lws_det_lat_plot_cb(struct lws_context *context, const lws_detlat_t *d);
-
-/**
- * lws_det_lat_active() - indicates if latencies are being measured
- *
- * \context: lws_context
- *
- * Returns 0 if latency measurement has not been set up (the callback is NULL).
- * Otherwise returns 1
- */
-LWS_VISIBLE LWS_EXTERN int
-lws_det_lat_active(struct lws_context *context);

include/libwebsockets/lws-lejp.h

@@ -182,7 +182,7 @@ typedef signed char (*lejp_callback)(struct lejp_ctx *ctx, char reason);
 #define LEJP_MAX_DEPTH 12
 #endif
 #ifndef LEJP_MAX_INDEX_DEPTH
-#define LEJP_MAX_INDEX_DEPTH 6
+#define LEJP_MAX_INDEX_DEPTH 8
 #endif
 #ifndef LEJP_MAX_PATH
 #define LEJP_MAX_PATH 128

include/libwebsockets/lws-metrics.h

@@ -0,0 +1,329 @@
+ /*
+ * libwebsockets - small server side websockets and web server implementation
+ *
+ * Copyright (C) 2010 - 2021 Andy Green <andy@warmcat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Public apis related to metric collection and reporting
+ */
+
+/* lws_metrics public part */
+
+typedef uint64_t u_mt_t;
+
+enum {
+	LWSMTFL_REPORT_OUTLIERS				= (1 << 0),
+	/**< track outliers and report them internally */
+	LWSMTFL_REPORT_OOB				= (1 << 1),
+	/**< report events as they happen */
+	LWSMTFL_REPORT_INACTIVITY_AT_PERIODIC		= (1 << 2),
+	/**< explicitly externally report no activity at periodic cb, by
+	 * default no events in the period is just not reported */
+	LWSMTFL_REPORT_MEAN				= (1 << 3),
+	/**< average/min/max is meaningful, else only sum is meaningful */
+	LWSMTFL_REPORT_ONLY_GO				= (1 << 4),
+	/**< no-go pieces invalid */
+	LWSMTFL_REPORT_DUTY_WALLCLOCK_US		= (1 << 5),
+	/**< aggregate compares to wallclock us for duty cycle */
+	LWSMTFL_REPORT_HIST				= (1 << 6),
+	/**< our type is histogram (otherwise, sum / mean aggregation) */
+};
+
+/*
+ * lws_metrics_tag allows your object to accumulate OpenMetrics-style
+ * descriptive tags before accounting for it with a metrics object at the end.
+ *
+ * Tags should represent low entropy information that is likely to repeat
+ * identically, so, eg, http method name, not eg, latency in us which is
+ * unlikely to be seen the same twice.
+ *
+ * Tags are just a list of name=value pairs, used for qualifying the final
+ * metrics entry with decorations in additional dimensions.  For example,
+ * rather than keep individual metrics on methods, scheme, mountpoint, result
+ * code, you can keep metrics on http transactions only, and qualify the
+ * transaction metrics entries with tags that can be queried on the metrics
+ * backend to get the finer-grained information.
+ *
+ * http_srv{code="404",mount="/",method="GET",scheme="http"} 3
+ *
+ * For OpenMetrics the tags are converted to a { list } and appended to the base
+ * metrics name before using with actual metrics objects, the same set of tags
+ * on different transactions resolve to the same qualification string.
+ */
+
+typedef struct lws_metrics_tag {
+	lws_dll2_t	list;
+
+	const char	*name; /* tag, intended to be in .rodata, not copied */
+	/* overallocated value */
+} lws_metrics_tag_t;
+
+LWS_EXTERN LWS_VISIBLE int
+lws_metrics_tag_add(lws_dll2_owner_t *owner, const char *name, const char *val);
+
+#if defined(LWS_WITH_SYS_METRICS)
+/*
+ * wsi-specific version that also appends the tag value to the lifecycle tag
+ * used for logging the wsi identity
+ */
+LWS_EXTERN LWS_VISIBLE int
+lws_metrics_tag_wsi_add(struct lws *wsi, const char *name, const char *val);
+#else
+#define lws_metrics_tag_wsi_add(_a, _b, _c)
+#endif
+
+#if defined(LWS_WITH_SECURE_STREAMS)
+/*
+ * ss-specific version that also appends the tag value to the lifecycle tag
+ * used for logging the ss identity
+ */
+#if defined(LWS_WITH_SYS_METRICS)
+LWS_EXTERN LWS_VISIBLE int
+lws_metrics_tag_ss_add(struct lws_ss_handle *ss, const char *name, const char *val);
+#else
+#define lws_metrics_tag_ss_add(_a, _b, _c)
+#endif
+#endif
+
+LWS_EXTERN LWS_VISIBLE void
+lws_metrics_tags_destroy(lws_dll2_owner_t *owner);
+
+LWS_EXTERN LWS_VISIBLE size_t
+lws_metrics_tags_serialize(lws_dll2_owner_t *owner, char *buf, size_t len);
+
+LWS_EXTERN LWS_VISIBLE const char *
+lws_metrics_tag_get(lws_dll2_owner_t *owner, const char *name);
+
+/* histogram bucket */
+
+typedef struct lws_metric_bucket {
+	struct lws_metric_bucket	*next;
+	uint64_t			count;
+
+	/* name + NUL is overallocated */
+} lws_metric_bucket_t;
+
+/* get overallocated name of bucket from bucket pointer */
+#define lws_metric_bucket_name_len(_b) (*((uint8_t *)&(_b)[1]))
+#define lws_metric_bucket_name(_b) (((const char *)&(_b)[1]) + 1)
+
+/*
+ * These represent persistent local event measurements.  They may aggregate
+ * a large number of events inbetween external dumping of summaries of the
+ * period covered, in two different ways
+ *
+ * 1) aggregation by sum or mean, to absorb multiple scalar readings
+ *
+ *  - go / no-go ratio counting
+ *  - mean averaging for, eg, latencies
+ *  - min / max for averaged values
+ *  - period the stats covers
+ *
+ * 2) aggregation by histogram, to absorb a range of outcomes that may occur
+ *    multiple times
+ *
+ *  - add named buckets to histogram
+ *  - bucket has a 64-bit count
+ *  - bumping a bucket just increments the count if already exists, else adds
+ *    a new one with count set to 1
+ *
+ * The same type with a union covers both cases.
+ *
+ * The lws_system ops api that hooks lws_metrics up to a metrics backend is
+ * given a pointer to these according to the related policy, eg, hourly, or
+ * every event passed straight through.
+ */
+
+typedef struct lws_metric_pub {
+	const char		*name;
+	/**< eg, "n.cn.dns", "vh.myendpoint" */
+	void			*backend_opaque;
+	/**< ignored by lws, backend handler completely owns it */
+
+	lws_usec_t		us_first;
+	/**< us time metric started collecting, reset to us_dumped at dump */
+	lws_usec_t		us_last;
+	/**< 0, or us time last event, reset to 0 at last dump */
+	lws_usec_t		us_dumped;
+	/**< 0 if never, else us time of last dump to external api */
+
+	/* scope of data in .u is "since last dump" --> */
+
+	union {
+		/* aggregation, by sum or mean */
+
+		struct {
+			u_mt_t			sum[2];
+			/**< go, no-go summed for mean or plan sum */
+			u_mt_t			min;
+			/**< smallest individual measurement */
+			u_mt_t			max;
+			/**< largest individual measurement */
+
+			uint32_t		count[2];
+			/**< go, no-go count of measurements in sum */
+		} agg;
+
+		/* histogram with dynamic named buckets */
+
+		struct {
+			lws_metric_bucket_t	*head;
+			/**< first bucket in our bucket list */
+
+			uint64_t		total_count;
+			/**< total count in all of our buckets */
+			uint32_t		list_size;
+			/**< number of buckets in our bucket list */
+		} hist;
+	} u;
+
+	uint8_t			flags;
+
+} lws_metric_pub_t;
+
+LWS_EXTERN LWS_VISIBLE void
+lws_metrics_hist_bump_priv_tagged(lws_metric_pub_t *mt, lws_dll2_owner_t *tow,
+				  lws_dll2_owner_t *tow2);
+
+
+/*
+ * Calipers are a helper struct for implementing "hanging latency" detection,
+ * where setting the start time and finding the end time may happen in more than
+ * one place.
+ *
+ * There are convenience wrappers to eliminate caliper definitions and code
+ * cleanly if WITH_SYS_METRICS is disabled for the build.
+ */
+
+struct lws_metric;
+
+typedef struct lws_metric_caliper {
+	struct lws_dll2_owner	mtags_owner; /**< collect tags here during
+					      * caliper lifetime */
+	struct lws_metric	*mt; /**< NULL == inactive */
+	lws_usec_t		us_start;
+} lws_metric_caliper_t;
+
+#if defined(LWS_WITH_SYS_METRICS)
+#define lws_metrics_caliper_compose(_name) \
+		lws_metric_caliper_t _name;
+#define lws_metrics_caliper_bind(_name, _mt) \
+	{ if (_name.mt) { \
+		lwsl_err("caliper: overwrite %s\n", \
+				lws_metrics_priv_to_pub(_name.mt)->name); \
+		assert(0); } \
+	  _name.mt = _mt; _name.us_start = lws_now_usecs(); }
+#define lws_metrics_caliper_declare(_name, _mt) \
+	lws_metric_caliper_t _name = { .mt = _mt, .us_start = lws_now_usecs() }
+#define lws_metrics_caliper_report(_name, _go_nogo) \
+	{ if (_name.us_start) { lws_metric_event(_name.mt, _go_nogo, \
+			   (u_mt_t)(lws_now_usecs() - \
+					   _name.us_start)); \
+					  }  lws_metrics_caliper_done(_name);  }
+#define lws_metrics_caliper_report_hist(_name, pwsi) if (_name.mt) { \
+		lws_metrics_hist_bump_priv_tagged(lws_metrics_priv_to_pub(_name.mt), \
+						  &_name.mtags_owner, \
+						  pwsi ? &((pwsi)->cal_conn.mtags_owner) : NULL); \
+		lws_metrics_caliper_done(_name);  }
+
+#define lws_metrics_caliper_cancel(_name) { lws_metrics_caliper_done(_name); }
+#define lws_metrics_hist_bump(_mt, _name) \
+		lws_metrics_hist_bump_(_mt, _name)
+#define lws_metrics_hist_bump_priv(_mt, _name) \
+		lws_metrics_hist_bump_(lws_metrics_priv_to_pub(_mt), _name)
+#define lws_metrics_caliper_done(_name) { \
+		_name.us_start = 0; _name.mt = NULL; \
+		lws_metrics_tags_destroy(&_name.mtags_owner); }
+#else
+#define lws_metrics_caliper_compose(_name)
+#define lws_metrics_caliper_bind(_name, _mt)
+#define lws_metrics_caliper_declare(_name, _mp)
+#define lws_metrics_caliper_report(_name, _go_nogo)
+#define lws_metrics_caliper_report_hist(_name, pwsiconn)
+#define lws_metrics_caliper_cancel(_name)
+#define lws_metrics_hist_bump(_mt, _name)
+#define lws_metrics_hist_bump_priv(_mt, _name)
+#define lws_metrics_caliper_done(_name)
+#endif
+
+/**
+ * lws_metrics_format() - helper to format a metrics object for logging
+ *
+ * \param pub: public part of metrics object
+ * \param buf: output buffer to place string in
+ * \param len: available length of \p buf
+ *
+ * Helper for describing the state of a metrics object as a human-readable
+ * string, accounting for how its flags indicate what it contains.  This is not
+ * how you would report metrics, but during development it can be useful to
+ * log them inbetween possibily long report intervals.
+ *
+ * It uses the metric's flags to adapt the format shown appropriately, eg,
+ * as a histogram if LWSMTFL_REPORT_HIST etc
+ */
+LWS_EXTERN LWS_VISIBLE int
+lws_metrics_format(lws_metric_pub_t *pub, lws_metric_bucket_t **sub,
+		   char *buf, size_t len);
+
+/**
+ * lws_metrics_hist_bump() - add or increment histogram bucket
+ *
+ * \param pub: public part of metrics object
+ * \param name: bucket name to increment
+ *
+ * Either increment the count of an existing bucket of the right name in the
+ * metrics object, or add a new bucket of the given name and set its count to 1.
+ *
+ * The metrics object must have been created with flag LWSMTFL_REPORT_HIST
+ *
+ * Normally, you will actually use the preprocessor wrapper
+ * lws_metrics_hist_bump() defined above, since this automatically takes care of
+ * removing itself from the build if WITH_SYS_METRICS is not defined, without
+ * needing any preprocessor conditionals.
+ */
+LWS_EXTERN LWS_VISIBLE int
+lws_metrics_hist_bump_(lws_metric_pub_t *pub, const char *name);
+
+LWS_VISIBLE LWS_EXTERN int
+lws_metrics_foreach(struct lws_context *ctx, void *user,
+		    int (*cb)(lws_metric_pub_t *pub, void *user));
+
+LWS_VISIBLE LWS_EXTERN int
+lws_metrics_hist_bump_describe_wsi(struct lws *wsi, lws_metric_pub_t *pub,
+				   const char *name);
+
+enum {
+	LMT_NORMAL = 0,	/* related to successful events */
+	LMT_OUTLIER,	/* related to successful events outside of bounds */
+
+	LMT_FAIL,	/* related to failed events */
+
+	LMT_COUNT,
+};
+
+typedef enum lws_metric_rpt {
+	LMR_PERIODIC = 0,	/* we are reporting on a schedule */
+	LMR_OUTLIER,		/* we are reporting the last outlier */
+} lws_metric_rpt_kind_t;
+
+#define METRES_GO	0
+#define METRES_NOGO	1
+
+

include/libwebsockets/lws-protocols-plugins.h

@@ -359,7 +359,22 @@ extern const struct lws_protocols lws_sshd_demo_protocols[1];
 extern const struct lws_protocols lws_acme_client_protocols[1];
 extern const struct lws_protocols client_loopback_test_protocols[1];
 extern const struct lws_protocols fulltext_demo_protocols[1];
+extern const struct lws_protocols lws_openmetrics_export_protocols[
+#if defined(LWS_WITH_SERVER) && defined(LWS_WITH_CLIENT) && defined(LWS_ROLE_WS)
+	4
+#else
+#if defined(LWS_WITH_SERVER)
+	3
+#else
+	1
+#endif
+#endif
+	];
 
+#define LWSOMPROIDX_DIRECT_HTTP_SERVER		0
+#define LWSOMPROIDX_PROX_HTTP_SERVER		1
+#define LWSOMPROIDX_PROX_WS_SERVER		2
+#define LWSOMPROIDX_PROX_WS_CLIENT		3
 
 #endif
 

include/libwebsockets/lws-secure-streams-policy.h

@@ -77,6 +77,25 @@ typedef struct lws_ss_plugin {
 } lws_ss_plugin_t;
 #endif
 
+/* the public, const metrics policy definition */
+
+typedef struct lws_metric_policy {
+	/* order of first two mandated by JSON policy parsing scope union */
+	const struct lws_metric_policy	*next;
+	const char			*name;
+
+	const char			*report;
+
+	/**< the metrics policy name in the policy, used to bind to it */
+	uint32_t			us_schedule;
+	/**< us interval between lws_system metrics api reports */
+
+	uint32_t			us_decay_unit;
+	/**< how many us to decay avg by half, 0 = no decay */
+	uint8_t				min_contributors;
+	/**< before we can judge something is an outlier */
+} lws_metric_policy_t;
+
 typedef struct lws_ss_x509 {
 	struct lws_ss_x509	*next;
 	const char		*vhost_name; /**< vhost name using cert ctx */
@@ -226,6 +245,7 @@ typedef struct lws_ss_policy {
 	const char		*payload_fmt;
 	const char		*socks5_proxy;
 	lws_ss_metadata_t	*metadata; /* linked-list of metadata */
+	const lws_metric_policy_t *metrics; /* linked-list of metric policies */
 	const lws_ss_auth_t	*auth; /* NULL or auth object we bind to */
 
 	/* protocol-specific connection policy details */

include/libwebsockets/lws-smd.h

@@ -52,6 +52,11 @@ enum {
 	 * Something happened on the network, eg, link-up or DHCP, or captive
 	 * portal state update
 	 */
+	LWSSMDCL_METRICS					= (1 << 3),
+	/**<
+	 * An SS client process is reporting a metric to the proxy (this class
+	 * is special in that it is not rebroadcast by the proxy)
+	 */
 
 	LWSSMDCL_USER_BASE_BITNUM				= 24
 };

include/libwebsockets/lws-stats.h

@@ -1,81 +0,0 @@
-/*
- * libwebsockets - small server side websockets and web server implementation
- *
- * Copyright (C) 2010 - 2019 Andy Green <andy@warmcat.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-
-/*
- * Stats are all uint64_t numbers that start at 0.
- * Index names here have the convention
- *
- *  _C_ counter
- *  _B_ byte count
- *  _MS_ millisecond count
- */
-
-enum {
-	LWSSTATS_C_CONNECTIONS, /**< count incoming connections */
-	LWSSTATS_C_API_CLOSE, /**< count calls to close api */
-	LWSSTATS_C_API_READ, /**< count calls to read from socket api */
-	LWSSTATS_C_API_LWS_WRITE, /**< count calls to lws_write API */
-	LWSSTATS_C_API_WRITE, /**< count calls to write API */
-	LWSSTATS_C_WRITE_PARTIALS, /**< count of partial writes */
-	LWSSTATS_C_WRITEABLE_CB_REQ, /**< count of writable callback requests */
-	LWSSTATS_C_WRITEABLE_CB_EFF_REQ, /**< count of effective writable callback requests */
-	LWSSTATS_C_WRITEABLE_CB, /**< count of writable callbacks */
-	LWSSTATS_C_SSL_CONNECTIONS_FAILED, /**< count of failed SSL connections */
-	LWSSTATS_C_SSL_CONNECTIONS_ACCEPTED, /**< count of accepted SSL connections */
-	LWSSTATS_C_SSL_ACCEPT_SPIN, /**< count of SSL_accept() attempts */
-	LWSSTATS_C_SSL_CONNS_HAD_RX, /**< count of accepted SSL conns that have had some RX */
-	LWSSTATS_C_TIMEOUTS, /**< count of timed-out connections */
-	LWSSTATS_C_SERVICE_ENTRY, /**< count of entries to lws service loop */
-	LWSSTATS_B_READ, /**< aggregate bytes read */
-	LWSSTATS_B_WRITE, /**< aggregate bytes written */
-	LWSSTATS_B_PARTIALS_ACCEPTED_PARTS, /**< aggreate of size of accepted write data from new partials */
-	LWSSTATS_US_SSL_ACCEPT_LATENCY_AVG, /**< aggregate delay in accepting connection */
-	LWSSTATS_US_WRITABLE_DELAY_AVG, /**< aggregate delay between asking for writable and getting cb */
-	LWSSTATS_US_WORST_WRITABLE_DELAY, /**< single worst delay between asking for writable and getting cb */
-	LWSSTATS_US_SSL_RX_DELAY_AVG, /**< aggregate delay between ssl accept complete and first RX */
-	LWSSTATS_C_PEER_LIMIT_AH_DENIED, /**< number of times we would have given an ah but for the peer limit */
-	LWSSTATS_C_PEER_LIMIT_WSI_DENIED, /**< number of times we would have given a wsi but for the peer limit */
-	LWSSTATS_C_CONNS_CLIENT, /**< attempted client conns */
-	LWSSTATS_C_CONNS_CLIENT_FAILED, /**< failed client conns */
-
-	/* Add new things just above here ---^
-	 * This is part of the ABI, don't needlessly break compatibility
-	 *
-	 * UPDATE stat_names in stats.c in sync with this!
-	 */
-	LWSSTATS_SIZE
-};
-
-#if defined(LWS_WITH_STATS)
-
-LWS_VISIBLE LWS_EXTERN uint64_t
-lws_stats_get(struct lws_context *context, int index);
-LWS_VISIBLE LWS_EXTERN void
-lws_stats_log_dump(struct lws_context *context);
-#else
-static LWS_INLINE uint64_t
-lws_stats_get(struct lws_context *context, int index) { (void)context; (void)index;  return 0; }
-static LWS_INLINE void
-lws_stats_log_dump(struct lws_context *context) { (void)context; }
-#endif

include/libwebsockets/lws-system.h

@@ -1,7 +1,7 @@
  /*
  * libwebsockets - small server side websockets and web server implementation
  *
- * Copyright (C) 2010 - 2020 Andy Green <andy@warmcat.com>
+ * Copyright (C) 2010 - 2021 Andy Green <andy@warmcat.com>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to
@@ -151,7 +151,6 @@ typedef enum {
 	LWS_CPD_NO_INTERNET,	/* we couldn't touch anything */
 } lws_cpd_result_t;
 
-
 typedef void (*lws_attach_cb_t)(struct lws_context *context, int tsi, void *opaque);
 struct lws_attach_item;
 
@@ -182,6 +181,11 @@ typedef struct lws_system_ops {
 	 * by calling lws_captive_portal_detect_result() api
 	 */
 
+	int (*metric_report)(lws_metric_pub_t *mdata);
+	/**< metric \p item is reporting an event of kind \p rpt,
+	 * held in \p mdata... return 0 to leave the metric object as it is,
+	 * or nonzero to reset it. */
+
 	uint32_t	wake_latency_us;
 	/**< time taken for this device to wake from suspend, in us
 	 */

lib/core-net/CMakeLists.txt

@@ -51,11 +51,6 @@ if (LWS_WITH_NETLINK)
 	)
 endif()
 
-if (LWS_WITH_DETAILED_LATENCY)
-	list(APPEND SOURCES
-		core-net/detailed-latency.c)
-endif()
-
 if (LWS_WITH_LWS_DSH)
 	list(APPEND SOURCES
 		core-net/lws-dsh.c)
@@ -77,20 +72,9 @@ if (LWS_WITH_CLIENT)
 	)
 endif()
 
-if (NOT LWS_WITHOUT_SERVER)
-	list(APPEND SOURCES
-		core-net/server.c)
-endif()
-
 if (LWS_WITH_SOCKS5 AND NOT LWS_WITHOUT_CLIENT)
 	list(APPEND SOURCES
 		core-net/socks5-client.c)
 endif()
 
-if (LWS_WITH_NETWORK AND LWS_WITH_STATS)
-	list(APPEND SOURCES
-		core-net/stats.c
-	)
-endif()
-
 exports_to_parent_scope()

lib/core-net/adopt.c

@@ -66,7 +66,11 @@ lws_create_new_server_wsi(struct lws_vhost *vhost, int fixed_tsi, const char *de
 		return NULL;
 	}
 
-	__lws_lc_tag(&vhost->context->lcg[LWSLCG_WSI_SERVER], &new_wsi->lc, desc);
+	__lws_lc_tag(&vhost->context->lcg[
+#if defined(LWS_ROLE_H2) || defined(LWS_ROLE_MQTT)
+	strcmp(desc, "adopted") ? LWSLCG_WSI_MUX :
+#endif
+	LWSLCG_WSI_SERVER], &new_wsi->lc, desc);
 
 	new_wsi->wsistate |= LWSIFR_SERVER;
 	new_wsi->tsi = (char)n;
@@ -77,11 +81,6 @@ lws_create_new_server_wsi(struct lws_vhost *vhost, int fixed_tsi, const char *de
 	new_wsi->rxflow_change_to = LWS_RXFLOW_ALLOW;
 	new_wsi->retry_policy = vhost->retry_policy;
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	if (vhost->context->detailed_latency_cb)
-		new_wsi->detlat.earliest_write_req_pre_write = lws_now_usecs();
-#endif
-
 	/* initialize the instance struct */
 
 	lwsi_set_state(new_wsi, LRS_UNCONNECTED);
@@ -145,8 +144,6 @@ lws_adopt_descriptor_vhost1(struct lws_vhost *vh, lws_adoption_type type,
 	pt = &context->pt[(int)new_wsi->tsi];
 	lws_pt_lock(pt, __func__);
 
-	lws_stats_bump(pt, LWSSTATS_C_CONNECTIONS, 1);
-
 	if (parent) {
 		new_wsi->parent = parent;
 		new_wsi->sibling_list = parent->child_list;
@@ -176,6 +173,11 @@ lws_adopt_descriptor_vhost1(struct lws_vhost *vh, lws_adoption_type type,
 		goto bail;
 	}
 
+#if defined(LWS_WITH_SERVER)
+	if (new_wsi->role_ops)
+		lws_metrics_tag_wsi_add(new_wsi, "role", new_wsi->role_ops->name);
+#endif
+
 	lws_pt_unlock(pt);
 
 	/*
@@ -494,9 +496,6 @@ lws_adopt_descriptor_vhost_via_info(const lws_adopt_desc_t *info)
 		    peer->count_wsi >= info->vh->context->ip_limit_wsi) {
 			lwsl_info("Peer reached wsi limit %d\n",
 					info->vh->context->ip_limit_wsi);
-			lws_stats_bump(&info->vh->context->pt[0],
-					      LWSSTATS_C_PEER_LIMIT_WSI_DENIED,
-					      1);
 			if (info->vh->context->pl_notify_cb)
 				info->vh->context->pl_notify_cb(
 							info->vh->context,

lib/core-net/client/connect.c

@@ -86,8 +86,12 @@ lws_client_connect_via_info(const struct lws_client_connect_info *i)
 	struct lws *wsi, *safe = NULL;
 	const struct lws_protocols *p;
 	const char *cisin[CIS_COUNT];
-	int tid = 0, n, tsi = 0;
 	struct lws_vhost *vh;
+	int
+#if LWS_MAX_SMP > 1
+		tid = 0,
+#endif
+		n, tsi = 0;
 	size_t size;
 	char *pc;
 
@@ -105,9 +109,6 @@ lws_client_connect_via_info(const struct lws_client_connect_info *i)
 	if (i->local_protocol_name)
 		local = i->local_protocol_name;
 
-	lws_stats_bump(&i->context->pt[tid], LWSSTATS_C_CONNS_CLIENT, 1);
-
-
 	lws_context_lock(i->context, __func__);
 	/*
 	 * PHASE 1: if SMP, find out the tsi related to current service thread
@@ -161,10 +162,6 @@ lws_client_connect_via_info(const struct lws_client_connect_info *i)
 		lws_fi_import(&wsi->fi, i->fi);
 #endif
 
-#if defined(LWS_WITH_DETAILED_LATENCY) && LWS_MAX_SMP > 1
-	wsi->detlat.tsi = tsi;
-#endif
-
 	/*
 	 * Until we exit, we can report connection failure directly to the
 	 * caller without needing to call through to protocol CONNECTION_ERROR.
@@ -186,11 +183,6 @@ lws_client_connect_via_info(const struct lws_client_connect_info *i)
 	else
 		wsi->retry_policy = &i->context->default_retry;
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	if (i->context->detailed_latency_cb)
-		wsi->detlat.earliest_write_req_pre_write = lws_now_usecs();
-#endif
-
 	if (i->ssl_connection & LCCSCF_WAKE_SUSPEND__VALIDITY)
 		wsi->conn_validity_wakesuspend = 1;
 
@@ -370,7 +362,8 @@ lws_client_connect_via_info(const struct lws_client_connect_info *i)
 			&wsi->lc, "%s/%s/%s/(%s)", i->method ? i->method : "WS",
 			wsi->role_ops->name, i->address,
 #if defined(LWS_WITH_SECURE_STREAMS_PROXY_API)
-			wsi->client_bound_sspc ? lws_sspc_tag((lws_sspc_handle_t *)i->opaque_user_data) :
+			wsi->client_bound_sspc ?
+				lws_sspc_tag((lws_sspc_handle_t *)i->opaque_user_data) :
 #endif
 			lws_ss_tag(((lws_ss_handle_t *)i->opaque_user_data)));
 	} else
@@ -379,6 +372,8 @@ lws_client_connect_via_info(const struct lws_client_connect_info *i)
 			     "%s/%s/%s", i->method ? i->method : "WS",
 			     wsi->role_ops->name, i->address);
 
+	lws_metrics_tag_wsi_add(wsi, "vh", wsi->a.vhost->name);
+
 	pc = (char *)&wsi->stash[1];
 
 	for (n = 0; n < CIS_COUNT; n++)
@@ -533,7 +528,5 @@ bail2:
 	if (i->pwsi)
 		*i->pwsi = NULL;
 
-	lws_stats_bump(&i->context->pt[tid], LWSSTATS_C_CONNS_CLIENT_FAILED, 1);
-
 	return NULL;
 }

lib/core-net/client/connect2.c

@@ -32,7 +32,11 @@
 static int
 lws_getaddrinfo46(struct lws *wsi, const char *ads, struct addrinfo **result)
 {
+	lws_metrics_caliper_declare(cal, wsi->a.context->mt_conn_dns);
 	struct addrinfo hints;
+#if defined(LWS_WITH_SYS_METRICS)
+	char buckname[32];
+#endif
 	int n;
 
 	memset(&hints, 0, sizeof(hints));
@@ -79,12 +83,26 @@ lws_getaddrinfo46(struct lws *wsi, const char *ads, struct addrinfo **result)
 
 #endif
 		wsi->dns_reachability = 1;
-		lwsl_notice("%s: asking to recheck CPD in 1ms\n", __func__);
+		lws_metrics_caliper_report(cal, METRES_NOGO);
-		lws_system_cpd_start_defer(wsi->a.context, LWS_US_PER_MS);
+#if defined(LWS_WITH_SYS_METRICS)
+		lws_snprintf(buckname, sizeof(buckname), "dns=\"unreachable %d\"", n);
+		lws_metrics_hist_bump_priv_wsi(wsi, mth_conn_failures, buckname);
+#endif
+		lwsl_notice("%s: asking to recheck CPD in 1s\n", __func__);
+		lws_system_cpd_start_defer(wsi->a.context, LWS_US_PER_SEC);
 	}
 
 	lwsl_info("%s: getaddrinfo '%s' says %d\n", __func__, ads, n);
 
+#if defined(LWS_WITH_SYS_METRICS)
+	if (n < 0) {
+		lws_snprintf(buckname, sizeof(buckname), "dns=\"nores %d\"", n);
+		lws_metrics_hist_bump_priv_wsi(wsi, mth_conn_failures, buckname);
+	}
+#endif
+
+	lws_metrics_caliper_report(cal, n >= 0 ? METRES_GO : METRES_NOGO);
+
 	return n;
 }
 #endif
@@ -260,19 +278,6 @@ solo:
 	}
 #endif
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	if (lwsi_state(wsi) == LRS_WAITING_DNS &&
-	    wsi->a.context->detailed_latency_cb) {
-		wsi->detlat.type = LDLT_NAME_RESOLUTION;
-		wsi->detlat.latencies[LAT_DUR_PROXY_CLIENT_REQ_TO_WRITE] =
-			(uint32_t)(lws_now_usecs() -
-			wsi->detlat.earliest_write_req_pre_write);
-		wsi->detlat.latencies[LAT_DUR_USERCB] = 0;
-		lws_det_lat_cb(wsi->a.context, &wsi->detlat);
-		wsi->detlat.earliest_write_req_pre_write = lws_now_usecs();
-	}
-#endif
-
 #if defined(LWS_CLIENT_HTTP_PROXYING) && \
 	(defined(LWS_ROLE_H1) || defined(LWS_ROLE_H2))
 
@@ -313,9 +318,6 @@ solo:
 	lwsl_info("%s: %s: lookup %s:%u\n", __func__, wsi->lc.gutag, ads, port);
 	wsi->conn_port = (uint16_t)port;
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	wsi->detlat.earliest_write_req_pre_write = lws_now_usecs();
-#endif
 #if !defined(LWS_WITH_SYS_ASYNC_DNS)
 	n = 0;
 	if (!wsi->dns_sorted_list.count) {

lib/core-net/client/connect3.c

@@ -203,6 +203,7 @@ lws_client_connect_3_connect(struct lws *wsi, const char *ads,
 		default:
 			lwsl_debug("%s: getsockopt check: conn fail: errno %d\n",
 					__func__, LWS_ERRNO);
+			lws_metrics_caliper_report(wsi->cal_conn, METRES_NOGO);
 			goto try_next_dns_result_fds;
 		}
 	}
@@ -236,19 +237,6 @@ lws_client_connect_3_connect(struct lws *wsi, const char *ads,
 	}
 #endif
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	if (lwsi_state(wsi) == LRS_WAITING_DNS &&
-	    wsi->a.context->detailed_latency_cb) {
-		wsi->detlat.type = LDLT_NAME_RESOLUTION;
-		wsi->detlat.latencies[LAT_DUR_PROXY_CLIENT_REQ_TO_WRITE] =
-			(uint32_t)(lws_now_usecs() -
-			wsi->detlat.earliest_write_req_pre_write);
-		wsi->detlat.latencies[LAT_DUR_USERCB] = 0;
-		lws_det_lat_cb(wsi->a.context, &wsi->detlat);
-		wsi->detlat.earliest_write_req_pre_write = lws_now_usecs();
-	}
-#endif
-
 	/*
 	 * Let's try directly connecting to each of the results in turn until
 	 * one works, or we run out of results...
@@ -393,11 +381,6 @@ ads_known:
 	 * The actual connection attempt
 	 */
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	wsi->detlat.earliest_write_req =
-		wsi->detlat.earliest_write_req_pre_write = lws_now_usecs();
-#endif
-
 #if defined(LWS_ESP_PLATFORM)
 	errno = 0;
 #endif
@@ -412,6 +395,12 @@ ads_known:
 	 * Finally, make the actual connection attempt
 	 */
 
+#if defined(LWS_WITH_SYS_METRICS)
+	if (wsi->cal_conn.mt)
+		lws_metrics_caliper_report(wsi->cal_conn, METRES_NOGO);
+	lws_metrics_caliper_bind(wsi->cal_conn, wsi->a.context->mt_conn_tcp);
+#endif
+
 	m = connect(wsi->desc.sockfd, (const struct sockaddr *)psa, (unsigned int)n);
 	if (m == -1) {
 		/*
@@ -438,6 +427,8 @@ ads_known:
 			 * The connect() failed immediately...
 			 */
 
+			lws_metrics_caliper_report(wsi->cal_conn, METRES_NOGO);
+
 #if defined(_DEBUG)
 #if defined(LWS_WITH_UNIX_SOCK)
 			if (!wsi->unix_skt) {
@@ -503,7 +494,12 @@ conn_good:
 				&salen) == -1)
 			lwsl_warn("getsockname: %s\n", strerror(LWS_ERRNO));
 #if defined(_DEBUG)
-		lws_sa46_write_numeric_address(&wsi->sa46_local, buf, sizeof(buf));
+#if defined(LWS_WITH_UNIX_SOCK)
+		if (wsi->unix_skt)
+			buf[0] = '\0';
+		else
+#endif
+			lws_sa46_write_numeric_address(&wsi->sa46_local, buf, sizeof(buf));
 
 		lwsl_info("%s: %s: source ads %s\n", __func__, wsi->lc.gutag, buf);
 #endif
@@ -511,20 +507,7 @@ conn_good:
 #endif
 
 	lws_sul_cancel(&wsi->sul_connect_timeout);
-
+	lws_metrics_caliper_report(wsi->cal_conn, METRES_GO);
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	if (wsi->a.context->detailed_latency_cb) {
-		wsi->detlat.type = LDLT_CONNECTION;
-		wsi->detlat.latencies[LAT_DUR_PROXY_CLIENT_REQ_TO_WRITE] =
-			(uint32_t)(lws_now_usecs() -
-			wsi->detlat.earliest_write_req_pre_write);
-		wsi->detlat.latencies[LAT_DUR_USERCB] = 0;
-		lws_det_lat_cb(wsi->a.context, &wsi->detlat);
-		wsi->detlat.earliest_write_req =
-			wsi->detlat.earliest_write_req_pre_write =
-							lws_now_usecs();
-	}
-#endif
 
 	lws_addrinfo_clean(wsi);
 
@@ -550,6 +533,8 @@ oom4:
 		/* do the full wsi close flow */
 		goto failed1;
 
+	lws_metrics_caliper_report(wsi->cal_conn, METRES_NOGO);
+
 	/*
 	 * We can't be an active client connection any more, if we thought
 	 * that was what we were going to be doing.  It should be if we are

lib/core-net/client/connect4.c

@@ -156,11 +156,7 @@ send_hs:
 		 * wait in the queue until it's possible to send them.
 		 */
 		lws_callback_on_writable(wsi_piggyback);
-#if defined(LWS_WITH_DETAILED_LATENCY)
+
-		wsi->detlat.earliest_write_req =
-				wsi->detlat.earliest_write_req_pre_write =
-								lws_now_usecs();
-#endif
 		lwsl_info("%s: %s: waiting to send hdrs (par state 0x%x)\n",
 			    __func__, wsi->lc.gutag, lwsi_state(wsi_piggyback));
 	} else {

lib/core-net/close.c

@@ -238,8 +238,6 @@ lws_inform_client_conn_fail(struct lws *wsi, void *arg, size_t len)
 		return;
 
 	wsi->already_did_cce = 1;
-	lws_stats_bump(&wsi->a.context->pt[(int)wsi->tsi],
-		       LWSSTATS_C_CONNS_CLIENT_FAILED, 1);
 
 	if (!wsi->a.protocol)
 		return;
@@ -293,6 +291,20 @@ __lws_close_free_wsi(struct lws *wsi, enum lws_close_status reason,
 	context = wsi->a.context;
 	pt = &context->pt[(int)wsi->tsi];
 
+#if defined(LWS_WITH_SYS_METRICS) && \
+    (defined(LWS_WITH_CLIENT) || defined(LWS_WITH_SERVER))
+	/* wsi level: only reports if dangling caliper */
+	if (wsi->cal_conn.mt && wsi->cal_conn.us_start) {
+		if ((lws_metrics_priv_to_pub(wsi->cal_conn.mt)->flags) & LWSMTFL_REPORT_HIST) {
+			lws_metrics_caliper_report_hist(wsi->cal_conn, (struct lws *)NULL);
+		} else {
+			lws_metrics_caliper_report(wsi->cal_conn, METRES_NOGO);
+			lws_metrics_caliper_done(wsi->cal_conn);
+		}
+	} else
+		lws_metrics_caliper_done(wsi->cal_conn);
+#endif
+
 #if defined(LWS_WITH_SYS_ASYNC_DNS)
 	if (wsi == context->async_dns.wsi)
 		context->async_dns.wsi = NULL;
@@ -300,8 +312,6 @@ __lws_close_free_wsi(struct lws *wsi, enum lws_close_status reason,
 
 	lws_pt_assert_lock_held(pt);
 
-	lws_stats_bump(pt, LWSSTATS_C_API_CLOSE, 1);
-
 #if defined(LWS_WITH_CLIENT)
 
 	lws_free_set_NULL(wsi->cli_hostname_copy);
@@ -719,6 +729,14 @@ async_close:
 				lws_sspc_handle_t *h = (lws_sspc_handle_t *)wsi->a.opaque_user_data;
 
 				if (h) { // && (h->info.flags & LWSSSINFLAGS_ACCEPTED)) {
+
+#if defined(LWS_WITH_SYS_METRICS)
+					/*
+					 * If any hanging caliper measurement, dump it, and free any tags
+					 */
+					lws_metrics_caliper_report_hist(h->cal_txn, (struct lws *)NULL);
+#endif
+
 					h->cwsi = NULL;
 					//wsi->a.opaque_user_data = NULL;
 				}
@@ -729,6 +747,12 @@ async_close:
 
 			if (h) { // && (h->info.flags & LWSSSINFLAGS_ACCEPTED)) {
 
+				/*
+				 * ss level: only reports if dangling caliper
+				 * not already reported
+				 */
+				lws_metrics_caliper_report_hist(h->cal_txn, wsi);
+
 				h->wsi = NULL;
 				wsi->a.opaque_user_data = NULL;
 

lib/core-net/detailed-latency.c

@@ -1,79 +0,0 @@
-/*
- * libwebsockets - small server side websockets and web server implementation
- *
- * Copyright (C) 2010 - 2019 Andy Green <andy@warmcat.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-
-#include "private-lib-core.h"
-
-int
-lws_det_lat_active(struct lws_context *context)
-{
-	return !!context->detailed_latency_cb;
-}
-
-int
-lws_det_lat_cb(struct lws_context *context, lws_detlat_t *d)
-{
-	int n;
-
-	if (!context->detailed_latency_cb)
-		return 0;
-
-	n = context->detailed_latency_cb(context, d);
-
-	memset(&d->latencies, 0, sizeof(d->latencies));
-
-	return n;
-}
-
-static const char types[] = "rwNCTt????";
-int
-lws_det_lat_plot_cb(struct lws_context *context, const lws_detlat_t *d)
-{
-	char buf[80], *p = buf, *end = &p[sizeof(buf) - 1];
-
-	if (!context->detailed_latency_filepath)
-		return 1;
-
-	if (context->latencies_fd == -1) {
-		context->latencies_fd = open(context->detailed_latency_filepath,
-				LWS_O_CREAT | LWS_O_TRUNC | LWS_O_WRONLY, 0600);
-		if (context->latencies_fd == -1)
-			return 1;
-	}
-
-	p += lws_snprintf(p, lws_ptr_diff_size_t(end, p),
-			  "%llu %c %u %u %u %u %u %zu %zu\n",
-			  (unsigned long long)lws_now_usecs(), types[d->type],
-			  d->latencies[LAT_DUR_PROXY_CLIENT_REQ_TO_WRITE],
-			  d->latencies[LAT_DUR_PROXY_CLIENT_WRITE_TO_PROXY_RX],
-			  d->latencies[LAT_DUR_PROXY_PROXY_REQ_TO_WRITE],
-			  d->latencies[LAT_DUR_PROXY_CLIENT_REQ_TO_WRITE] +
-			  d->latencies[LAT_DUR_PROXY_CLIENT_WRITE_TO_PROXY_RX] +
-			  d->latencies[LAT_DUR_PROXY_PROXY_REQ_TO_WRITE],
-			  d->latencies[LAT_DUR_PROXY_RX_TO_ONWARD_TX],
-			  d->acc_size, d->req_size);
-
-	write(context->latencies_fd, buf, lws_ptr_diff_size_t(p, buf));
-
-	return 0;
-}

lib/core-net/network.c

@@ -890,7 +890,7 @@ lws_sa46_write_numeric_address(lws_sockaddr46 *sa46, char *buf, size_t len)
 		return lws_snprintf(buf, len, "(unset)");
 
 	if (sa46->sa4.sin_family == AF_INET6)
-		lws_snprintf(buf, len, "(ipv6 unsupp)");
+		return lws_snprintf(buf, len, "(ipv6 unsupp)");
 
 	lws_snprintf(buf, len, "(AF%d unsupp)", (int)sa46->sa4.sin_family);
 

lib/core-net/output.c

@@ -31,7 +31,6 @@ int
 lws_issue_raw(struct lws *wsi, unsigned char *buf, size_t len)
 {
 	struct lws_context *context = lws_get_context(wsi);
-	struct lws_context_per_thread *pt = &wsi->a.context->pt[(int)wsi->tsi];
 	size_t real_len = len;
 	unsigned int n, m;
 
@@ -59,8 +58,6 @@ lws_issue_raw(struct lws *wsi, unsigned char *buf, size_t len)
 			  (unsigned long)len);
 	}
 
-	lws_stats_bump(pt, LWSSTATS_C_API_WRITE, 1);
-
 	/* just ignore sends after we cleared the truncation buffer */
 	if (lwsi_state(wsi) == LRS_FLUSHING_BEFORE_CLOSE &&
 	    !lws_has_buffered_out(wsi)
@@ -215,9 +212,6 @@ lws_issue_raw(struct lws *wsi, unsigned char *buf, size_t len)
 				       real_len - m) < 0)
 		return -1;
 
-	lws_stats_bump(pt, LWSSTATS_C_WRITE_PARTIALS, 1);
-	lws_stats_bump(pt, LWSSTATS_B_PARTIALS_ACCEPTED_PARTS, m);
-
 #if defined(LWS_WITH_UDP)
 	if (lws_wsi_is_udp(wsi))
 		/* stash original destination for fulfilling UDP partials */
@@ -234,57 +228,30 @@ int
 lws_write(struct lws *wsi, unsigned char *buf, size_t len,
 	  enum lws_write_protocol wp)
 {
-	struct lws_context_per_thread *pt = &wsi->a.context->pt[(int)wsi->tsi];
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	lws_usec_t us;
-#endif
 	int m;
 
-	lws_stats_bump(pt, LWSSTATS_C_API_LWS_WRITE, 1);
-
 	if ((int)len < 0) {
 		lwsl_err("%s: suspicious len int %d, ulong %lu\n", __func__,
 				(int)len, (unsigned long)len);
 		return -1;
 	}
 
-	lws_stats_bump(pt, LWSSTATS_B_WRITE, len);
-
 #ifdef LWS_WITH_ACCESS_LOG
 	wsi->http.access_log.sent += len;
 #endif
-#if defined(LWS_WITH_SERVER_STATUS)
-	if (wsi->a.vhost)
-		wsi->a.vhost->conn_stats.tx += len;
-#endif
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	us = lws_now_usecs();
-#endif
 
 	assert(wsi->role_ops);
 
 	if (!lws_rops_fidx(wsi->role_ops, LWS_ROPS_write_role_protocol))
-		return lws_issue_raw(wsi, buf, len);
+		m = lws_issue_raw(wsi, buf, len);
+	else
+		m = lws_rops_func_fidx(wsi->role_ops, LWS_ROPS_write_role_protocol).
+				write_role_protocol(wsi, buf, len, &wp);
 
-	m = lws_rops_func_fidx(wsi->role_ops, LWS_ROPS_write_role_protocol).
+#if defined(LWS_WITH_SYS_METRICS)
-			write_role_protocol(wsi, buf, len, &wp);
+	if (wsi->a.vhost)
-	if (m < 0)
+		lws_metric_event(wsi->a.vhost->mt_traffic_tx, (char)
-		return m;
+				 (m < 0 ? METRES_NOGO : METRES_GO), len);
-
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	if (wsi->a.context->detailed_latency_cb) {
-		wsi->detlat.req_size = len;
-		wsi->detlat.acc_size = (unsigned int)m;
-		wsi->detlat.type = LDLT_WRITE;
-		if (wsi->detlat.earliest_write_req_pre_write)
-			wsi->detlat.latencies[LAT_DUR_PROXY_PROXY_REQ_TO_WRITE] =
-					(uint32_t)(us - wsi->detlat.earliest_write_req_pre_write);
-		else
-			wsi->detlat.latencies[LAT_DUR_PROXY_PROXY_REQ_TO_WRITE] = 0;
-		wsi->detlat.latencies[LAT_DUR_USERCB] = (uint32_t)(lws_now_usecs() - us);
-		lws_det_lat_cb(wsi->a.context, &wsi->detlat);
-
-	}
 #endif
 
 	return m;
@@ -316,19 +283,20 @@ lws_ssl_capable_read_no_ssl(struct lws *wsi, unsigned char *buf, size_t len)
 	en = LWS_ERRNO;
 	if (n >= 0) {
 
-		if (!n && wsi->unix_skt)
+		//if (!n && wsi->unix_skt)
-			return LWS_SSL_CAPABLE_ERROR;
+		//	goto do_err;
 
 		/*
 		 * See https://libwebsockets.org/
 		 * pipermail/libwebsockets/2019-March/007857.html
 		 */
-		if (!n)
+		if (!n && !wsi->unix_skt)
-			return LWS_SSL_CAPABLE_ERROR;
+			goto do_err;
 
-#if defined(LWS_WITH_SERVER_STATUS)
+#if defined(LWS_WITH_SYS_METRICS) && defined(LWS_WITH_SERVER)
 		if (wsi->a.vhost)
-			wsi->a.vhost->conn_stats.rx = (unsigned long long)(wsi->a.vhost->conn_stats.rx + (unsigned long long)(long long)n);
+			lws_metric_event(wsi->a.vhost->mt_traffic_rx,
+					 METRES_GO /* rx */, (unsigned int)n);
 #endif
 
 		return n;
@@ -339,7 +307,14 @@ lws_ssl_capable_read_no_ssl(struct lws *wsi, unsigned char *buf, size_t len)
 	    en == LWS_EINTR)
 		return LWS_SSL_CAPABLE_MORE_SERVICE;
 
-	lwsl_info("error on reading from skt : %d\n", en);
+do_err:
+#if defined(LWS_WITH_SYS_METRICS) && defined(LWS_WITH_SERVER)
+	if (wsi->a.vhost)
+		lws_metric_event(wsi->a.vhost->mt_traffic_rx, METRES_NOGO, 0u);
+#endif
+
+	lwsl_info("%s: error on reading from skt : %d, errno %d\n",
+			__func__, n, en);
 
 	return LWS_SSL_CAPABLE_ERROR;
 }

lib/core-net/pollfd.c

@@ -515,7 +515,6 @@ lws_change_pollfd(struct lws *wsi, int _and, int _or)
 int
 lws_callback_on_writable(struct lws *wsi)
 {
-	struct lws_context_per_thread *pt;
 	struct lws *w = wsi;
 
 	if (lwsi_state(wsi) == LRS_SHUTDOWN)
@@ -524,21 +523,6 @@ lws_callback_on_writable(struct lws *wsi)
 	if (wsi->socket_is_permanently_unusable)
 		return 0;
 
-	pt = &wsi->a.context->pt[(int)wsi->tsi];
-
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	if (!wsi->detlat.earliest_write_req)
-		wsi->detlat.earliest_write_req = lws_now_usecs();
-#endif
-
-	lws_stats_bump(pt, LWSSTATS_C_WRITEABLE_CB_REQ, 1);
-#if defined(LWS_WITH_STATS)
-	if (!wsi->active_writable_req_us) {
-		wsi->active_writable_req_us = lws_now_usecs();
-		lws_stats_bump(pt, LWSSTATS_C_WRITEABLE_CB_EFF_REQ, 1);
-	}
-#endif
-
 	if (lws_rops_fidx(wsi->role_ops, LWS_ROPS_callback_on_writable)) {
 		int q = lws_rops_func_fidx(wsi->role_ops,
 					   LWS_ROPS_callback_on_writable).

lib/core-net/private-lib-core-net.h

@@ -264,41 +264,6 @@ struct lws_timed_vh_protocol {
 
 #endif
 
-/*
- * lws_dsh
-*/
-
-typedef struct lws_dsh_obj_head {
-	lws_dll2_owner_t		owner;
-	size_t				total_size; /* for this kind in dsh */
-	int				kind;
-} lws_dsh_obj_head_t;
-
-typedef struct lws_dsh_obj {
-	lws_dll2_t			list;	/* must be first */
-	struct lws_dsh	  		*dsh;	/* invalid when on free list */
-	size_t				size;	/* invalid when on free list */
-	size_t				asize;
-	int				kind; /* so we can account at free */
-} lws_dsh_obj_t;
-
-typedef struct lws_dsh {
-	lws_dll2_t			list;
-	uint8_t				*buf;
-	lws_dsh_obj_head_t		*oha;	/* array of object heads/kind */
-	size_t				buffer_size;
-	size_t				locally_in_use;
-	size_t				locally_free;
-	int				count_kinds;
-	uint8_t				being_destroyed;
-	/*
-	 * Overallocations at create:
-	 *
-	 *  - the buffer itself
-	 *  - the object heads array
-	 */
-} lws_dsh_t;
-
 /*
  * lws_async_dns
  */
@@ -371,11 +336,6 @@ struct lws_context_per_thread {
 #if defined(LWS_ROLE_CGI)
 	lws_sorted_usec_list_t sul_cgi;
 #endif
-#if defined(LWS_WITH_STATS)
-	uint64_t lws_stats[LWSSTATS_SIZE];
-	int updated;
-	lws_sorted_usec_list_t sul_stats;
-#endif
 #if defined(LWS_WITH_PEER_LIMITS)
 	lws_sorted_usec_list_t sul_peer_limits;
 #endif
@@ -419,10 +379,6 @@ struct lws_context_per_thread {
 	void		*evlib_pt; /* overallocated */
 #endif
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	lws_usec_t	ust_left_poll;
-#endif
-
 	/* --- */
 
 	unsigned long count_conns;
@@ -454,14 +410,6 @@ struct lws_context_per_thread {
 	unsigned char is_destroyed:1;
 };
 
-#if defined(LWS_WITH_SERVER_STATUS)
-struct lws_conn_stats {
-	unsigned long long rx, tx;
-	unsigned long h1_conn, h1_trans, h2_trans, ws_upg, h2_alpn, h2_subs,
-		      h2_upg, rejected, mqtt_subs;
-};
-#endif
-
 /*
  * virtual host -related context information
  *   vhostwide SSL context
@@ -510,8 +458,9 @@ struct lws_vhost {
 #if defined(LWS_WITH_EVENT_LIBS)
 	void		*evlib_vh; /* overallocated */
 #endif
-#if defined(LWS_WITH_SERVER_STATUS)
+#if defined(LWS_WITH_SYS_METRICS)
-	struct lws_conn_stats conn_stats;
+	lws_metric_t	*mt_traffic_rx;
+	lws_metric_t	*mt_traffic_tx;
 #endif
 
 #if defined(LWS_WITH_SYS_FAULT_INJECTION)
@@ -707,10 +656,6 @@ struct lws {
 	void				*evlib_wsi; /* overallocated */
 #endif
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	lws_detlat_t	detlat;
-#endif
-
 	lws_sorted_usec_list_t		sul_timeout;
 	lws_sorted_usec_list_t		sul_hrtimer;
 	lws_sorted_usec_list_t		sul_validity;
@@ -728,6 +673,8 @@ struct lws {
 	struct lws_dll2			dll2_cli_txn_queue;
 	struct lws_dll2_owner		dll2_cli_txn_queue_owner;
 
+	/**< caliper is reused for tcp, tls and txn conn phases */
+
 	lws_dll2_t			speculative_list;
 	lws_dll2_owner_t		speculative_connect_owner;
 	/* wsis: additional connection candidates */
@@ -741,6 +688,10 @@ struct lws {
 	/**< Fault Injection ctx for the wsi, hierarchy wsi->vhost->context */
 #endif
 
+#if defined(LWS_WITH_SYS_METRICS)
+	lws_metrics_caliper_compose(cal_conn)
+#endif
+
 	lws_sockaddr46			sa46_local;
 	lws_sockaddr46			sa46_peer;
 
@@ -779,12 +730,7 @@ struct lws {
 #endif
 
 	lws_sock_file_fd_type		desc; /* .filefd / .sockfd */
-#if defined(LWS_WITH_STATS)
+
-	uint64_t active_writable_req_us;
-#if defined(LWS_WITH_TLS)
-	uint64_t accept_start_us;
-#endif
-#endif
 	lws_wsi_state_t			wsistate;
 	lws_wsi_state_t			wsistate_pre_close;
 
@@ -909,9 +855,6 @@ struct lws {
 #if defined(LWS_WITH_CGI) || defined(LWS_WITH_CLIENT)
 	char reason_bf; /* internal writeable callback reason bitfield */
 #endif
-#if defined(LWS_WITH_STATS) && defined(LWS_WITH_TLS)
-	char seen_rx;
-#endif
 #if defined(LWS_WITH_NETLINK)
 	lws_route_uidx_t		peer_route_uidx;
 	/**< unique index of the route the connection is estimated to take */
@@ -1224,11 +1167,6 @@ lws_destroy_event_pipe(struct lws *wsi);
 int
 lws_socks5c_generate_msg(struct lws *wsi, enum socks_msg_type type, ssize_t *msg_len);
 
-#if defined(LWS_WITH_SERVER_STATUS)
-void
-lws_sum_stats(const struct lws_context *ctx, struct lws_conn_stats *cs);
-#endif
-
 #if defined(LWS_WITH_DEPRECATED_THINGS)
 int
 __lws_timed_callback_remove(struct lws_vhost *vh, struct lws_timed_vh_protocol *p);
@@ -1433,21 +1371,6 @@ lws_sort_dns(struct lws *wsi, const struct addrinfo *result);
 int
 lws_broadcast(struct lws_context_per_thread *pt, int reason, void *in, size_t len);
 
-#if defined(LWS_WITH_STATS)
- void
- lws_stats_bump(struct lws_context_per_thread *pt, int i, uint64_t bump);
- void
- lws_stats_max(struct lws_context_per_thread *pt, int index, uint64_t val);
-#else
- static LWS_INLINE uint64_t lws_stats_bump(
-		struct lws_context_per_thread *pt, int index, uint64_t bump) {
-	(void)pt; (void)index; (void)bump; return 0; }
- static LWS_INLINE uint64_t lws_stats_max(
-		struct lws_context_per_thread *pt, int index, uint64_t val) {
-	(void)pt; (void)index; (void)val; return 0; }
-#endif
-
-
 
 #if defined(LWS_WITH_PEER_LIMITS)
 void
@@ -1500,6 +1423,9 @@ extern const struct lws_protocols protocol_abs_client_raw_skt,
 void
 __lws_reset_wsi(struct lws *wsi);
 
+void
+lws_metrics_dump(struct lws_context *ctx);
+
 void
 lws_inform_client_conn_fail(struct lws *wsi, void *arg, size_t len);
 

lib/core-net/server.c

@@ -1,326 +0,0 @@
-/*
- * libwebsockets - small server side websockets and web server implementation
- *
- * Copyright (C) 2010 - 2019 Andy Green <andy@warmcat.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-
-#include "private-lib-core.h"
-
-#if defined(LWS_WITH_SERVER_STATUS)
-
-void
-lws_sum_stats(const struct lws_context *ctx, struct lws_conn_stats *cs)
-{
-	const struct lws_vhost *vh = ctx->vhost_list;
-
-	while (vh) {
-
-		cs->rx += vh->conn_stats.rx;
-		cs->tx += vh->conn_stats.tx;
-		cs->h1_conn += vh->conn_stats.h1_conn;
-		cs->h1_trans += vh->conn_stats.h1_trans;
-		cs->h2_trans += vh->conn_stats.h2_trans;
-		cs->ws_upg += vh->conn_stats.ws_upg;
-		cs->h2_upg += vh->conn_stats.h2_upg;
-		cs->h2_alpn += vh->conn_stats.h2_alpn;
-		cs->h2_subs += vh->conn_stats.h2_subs;
-		cs->rejected += vh->conn_stats.rejected;
-
-		vh = vh->vhost_next;
-	}
-}
-
-int
-lws_json_dump_vhost(const struct lws_vhost *vh, char *buf, int len)
-{
-#if defined(LWS_ROLE_H1) || defined(LWS_ROLE_H2)
-	static const char * const prots[] = {
-		"http://",
-		"https://",
-		"file://",
-		"cgi://",
-		">http://",
-		">https://",
-		"callback://"
-	};
-#endif
-	char *orig = buf, *end = buf + len - 1, first;
-	int n;
-
-	if (len < 100)
-		return 0;
-
-	buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf),
-			"{\n \"name\":\"%s\",\n"
-			" \"port\":\"%d\",\n"
-			" \"use_ssl\":\"%d\",\n"
-			" \"sts\":\"%d\",\n"
-			" \"rx\":\"%llu\",\n"
-			" \"tx\":\"%llu\",\n"
-			" \"h1_conn\":\"%lu\",\n"
-			" \"h1_trans\":\"%lu\",\n"
-			" \"h2_trans\":\"%lu\",\n"
-			" \"ws_upg\":\"%lu\",\n"
-			" \"rejected\":\"%lu\",\n"
-			" \"h2_upg\":\"%lu\",\n"
-			" \"h2_alpn\":\"%lu\",\n"
-			" \"h2_subs\":\"%lu\""
-			,
-			vh->name, vh->listen_port,
-#if defined(LWS_WITH_TLS)
-			vh->tls.use_ssl & LCCSCF_USE_SSL,
-#else
-			0,
-#endif
-			!!(vh->options & LWS_SERVER_OPTION_STS),
-			vh->conn_stats.rx, vh->conn_stats.tx,
-			vh->conn_stats.h1_conn,
-			vh->conn_stats.h1_trans,
-			vh->conn_stats.h2_trans,
-			vh->conn_stats.ws_upg,
-			vh->conn_stats.rejected,
-			vh->conn_stats.h2_upg,
-			vh->conn_stats.h2_alpn,
-			vh->conn_stats.h2_subs
-	);
-#if defined(LWS_ROLE_H1) || defined(LWS_ROLE_H2)
-	if (vh->http.mount_list) {
-		const struct lws_http_mount *m = vh->http.mount_list;
-
-		buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), ",\n \"mounts\":[");
-		first = 1;
-		while (m) {
-			if (!first)
-				buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), ",");
-			buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf),
-					"\n  {\n   \"mountpoint\":\"%s\",\n"
-					"  \"origin\":\"%s%s\",\n"
-					"  \"cache_max_age\":\"%d\",\n"
-					"  \"cache_reuse\":\"%d\",\n"
-					"  \"cache_revalidate\":\"%d\",\n"
-					"  \"cache_intermediaries\":\"%d\"\n"
-					,
-					m->mountpoint,
-					prots[m->origin_protocol],
-					m->origin,
-					m->cache_max_age,
-					m->cache_reusable,
-					m->cache_revalidate,
-					m->cache_intermediaries);
-			if (m->def)
-				buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf),
-						",\n  \"default\":\"%s\"",
-						m->def);
-			buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), "\n  }");
-			first = 0;
-			m = m->mount_next;
-		}
-		buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), "\n ]");
-	}
-#endif
-	if (vh->protocols) {
-		n = 0;
-		first = 1;
-
-		buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), ",\n \"ws-protocols\":[");
-		while (n < vh->count_protocols) {
-			if (!first)
-				buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), ",");
-			buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf),
-					"\n  {\n   \"%s\":{\n"
-					"    \"status\":\"ok\"\n   }\n  }"
-					,
-					vh->protocols[n].name);
-			first = 0;
-			n++;
-		}
-		buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), "\n ]");
-	}
-
-	buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), "\n}");
-
-	return lws_ptr_diff(buf, orig);
-}
-
-
-int
-lws_json_dump_context(const struct lws_context *context, char *buf, int len,
-		int hide_vhosts)
-{
-	char *orig = buf, *end = buf + len - 1, first = 1;
-	const struct lws_vhost *vh = context->vhost_list;
-	const struct lws_context_per_thread *pt;
-	int n, listening = 0, cgi_count = 0, fd;
-	struct lws_conn_stats cs;
-	double d = 0;
-#ifdef LWS_WITH_CGI
-	struct lws_cgi * const *pcgi;
-#endif
-
-//#ifdef LWS_WITH_LIBUV &&
-//	uv_uptime(&d);
-//#endif
-
-	buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), "{ "
-			    "\"version\":\"%s\",\n"
-			    "\"uptime\":\"%ld\",\n",
-			    lws_get_library_version(),
-			    (long)d);
-
-#ifdef LWS_HAVE_GETLOADAVG
-#if defined(__sun)
-#include <sys/loadavg.h>
-#endif
-	{
-		double d[3];
-		int m;
-
-		m = getloadavg(d, 3);
-		for (n = 0; n < m; n++) {
-			buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf),
-				"\"l%d\":\"%.2f\",\n",
-				n + 1, d[n]);
-		}
-	}
-#endif
-
-	fd = lws_open("/proc/self/statm", LWS_O_RDONLY);
-	if (fd >= 0) {
-		char contents[96], pure[96];
-		n = (int)read(fd, contents, sizeof(contents) - 1);
-		if (n > 0) {
-			contents[n] = '\0';
-			if (contents[n - 1] == '\n')
-				contents[--n] = '\0';
-			lws_json_purify(pure, contents, sizeof(pure), NULL);
-
-			buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf),
-					  "\"statm\": \"%s\",\n", pure);
-		}
-		close(fd);
-	}
-
-	buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), "\"heap\":%lld,\n\"contexts\":[\n",
-				(long long)lws_get_allocated_heap());
-
-	buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), "{ "
-				"\"context_uptime\":\"%llu\",\n"
-				"\"cgi_spawned\":\"%d\",\n"
-				"\"pt_fd_max\":\"%d\",\n"
-				"\"ah_pool_max\":\"%d\",\n"
-				"\"deprecated\":\"%d\",\n"
-				"\"wsi_alive\":\"",
-				(unsigned long long)(lws_now_usecs() - context->time_up) /
-					LWS_US_PER_SEC,
-				context->count_cgi_spawned,
-				context->fd_limit_per_thread,
-				context->max_http_header_pool,
-				context->deprecated);
-
-	for (n = 0; n < LWSLCG_COUNT; n++)
-		buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), "%u ",
-				context->lcg[n].owner.count);
-
-	buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), "\", \"pt\":[\n ");
-	for (n = 0; n < context->count_threads; n++) {
-		pt = &context->pt[n];
-		if (n)
-			buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), ",");
-		buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf),
-				"\n  {\n"
-				"    \"fds_count\":\"%d\",\n"
-				"    \"ah_pool_inuse\":\"%d\",\n"
-				"    \"ah_wait_list\":\"%d\"\n"
-				"    }",
-				pt->fds_count,
-				pt->http.ah_count_in_use,
-				pt->http.ah_wait_list_length);
-	}
-
-	buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), "]");
-
-	buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), ", \"vhosts\":[\n ");
-
-	first = 1;
-	vh = context->vhost_list;
-	listening = 0;
-	cs = context->conn_stats;
-	lws_sum_stats(context, &cs);
-	while (vh) {
-
-		if (!hide_vhosts) {
-			if (!first)
-				if (buf != end)
-					*buf++ = ',';
-			buf += lws_json_dump_vhost(vh, buf, lws_ptr_diff(end, buf));
-			first = 0;
-		}
-		if (vh->lserv_wsi)
-			listening++;
-		vh = vh->vhost_next;
-	}
-
-	buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf),
-			"],\n\"listen_wsi\":\"%d\",\n"
-			" \"rx\":\"%llu\",\n"
-			" \"tx\":\"%llu\",\n"
-			" \"h1_conn\":\"%lu\",\n"
-			" \"h1_trans\":\"%lu\",\n"
-			" \"h2_trans\":\"%lu\",\n"
-			" \"ws_upg\":\"%lu\",\n"
-			" \"rejected\":\"%lu\",\n"
-			" \"h2_alpn\":\"%lu\",\n"
-			" \"h2_subs\":\"%lu\",\n"
-			" \"h2_upg\":\"%lu\"",
-			listening, cs.rx, cs.tx,
-			cs.h1_conn,
-			cs.h1_trans,
-			cs.h2_trans,
-			cs.ws_upg,
-			cs.rejected,
-			cs.h2_alpn,
-			cs.h2_subs,
-			cs.h2_upg);
-
-#ifdef LWS_WITH_CGI
-	for (n = 0; n < context->count_threads; n++) {
-		pt = &context->pt[n];
-		pcgi = &pt->http.cgi_list;
-
-		while (*pcgi) {
-			pcgi = &(*pcgi)->cgi_list;
-
-			cgi_count++;
-		}
-	}
-#endif
-	buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), ",\n \"cgi_alive\":\"%d\"\n ",
-			cgi_count);
-
-	buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), "}");
-
-
-	buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), "]}\n ");
-
-	return lws_ptr_diff(buf, orig);
-}
-
-#endif

lib/core-net/service.c

@@ -27,31 +27,8 @@
 int
 lws_callback_as_writeable(struct lws *wsi)
 {
-	struct lws_context_per_thread *pt = &wsi->a.context->pt[(int)wsi->tsi];
 	int n, m;
 
-	lws_stats_bump(pt, LWSSTATS_C_WRITEABLE_CB, 1);
-#if defined(LWS_WITH_STATS)
-	if (wsi->active_writable_req_us) {
-		uint64_t ul = lws_now_usecs() -
-			      wsi->active_writable_req_us;
-
-		lws_stats_bump(pt, LWSSTATS_US_WRITABLE_DELAY_AVG, ul);
-		lws_stats_max(pt, LWSSTATS_US_WORST_WRITABLE_DELAY, ul);
-		wsi->active_writable_req_us = 0;
-	}
-#endif
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	if (wsi->a.context->detailed_latency_cb && lwsi_state_est(wsi)) {
-		lws_usec_t us = lws_now_usecs();
-
-		wsi->detlat.earliest_write_req_pre_write =
-					wsi->detlat.earliest_write_req;
-		wsi->detlat.earliest_write_req = 0;
-		wsi->detlat.latencies[LAT_DUR_PROXY_RX_TO_ONWARD_TX] =
-		      (uint32_t)(us - wsi->detlat.earliest_write_req_pre_write);
-	}
-#endif
 	n = wsi->role_ops->writeable_cb[lwsi_role_server(wsi)];
 	m = user_callback_handle_rxflow(wsi->a.protocol->callback,
 					wsi, (enum lws_callback_reasons) n,
@@ -827,7 +804,8 @@ lws_service(struct lws_context *context, int timeout_ms)
 	}
 	n = lws_plat_service(context, timeout_ms);
 
-	pt->inside_service = 0;
+	if (n != -1)
+		pt->inside_service = 0;
 
 	return n;
 }

lib/core-net/sorted-usec-list.c

@@ -147,6 +147,8 @@ __lws_sul_service_ripe(lws_dll2_owner_t *own, int own_len, lws_usec_t usnow)
 		lws_dll2_remove(&hit->list);
 		hit->us = 0;
 
+		// lwsl_notice("%s: sul: %p\n", __func__, hit->cb);
+
 		pt->inside_lws_service = 1;
 		hit->cb(hit);
 		pt->inside_lws_service = 0;

lib/core-net/stats.c

@@ -1,273 +0,0 @@
-/*
- * libwebsockets - small server side websockets and web server implementation
- *
- * Copyright (C) 2010 - 2019 Andy Green <andy@warmcat.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-
-#include "private-lib-core.h"
-
-#if defined(LWS_WITH_STATS)
-
-uint64_t
-lws_stats_get(struct lws_context *context, int index)
-{
-	struct lws_context_per_thread *pt = &context->pt[0];
-
-	if (index >= LWSSTATS_SIZE)
-		return 0;
-
-	return pt->lws_stats[index];
-}
-
-static const char * stat_names[] = {
-	"C_CONNECTIONS",
-	"C_API_CLOSE",
-	"C_API_READ",
-	"C_API_LWS_WRITE",
-	"C_API_WRITE",
-	"C_WRITE_PARTIALS",
-	"C_WRITEABLE_CB_REQ",
-	"C_WRITEABLE_CB_EFF_REQ",
-	"C_WRITEABLE_CB",
-	"C_SSL_CONNECTIONS_FAILED",
-	"C_SSL_CONNECTIONS_ACCEPTED",
-	"C_SSL_CONNECTIONS_ACCEPT_SPIN",
-	"C_SSL_CONNS_HAD_RX",
-	"C_TIMEOUTS",
-	"C_SERVICE_ENTRY",
-	"B_READ",
-	"B_WRITE",
-	"B_PARTIALS_ACCEPTED_PARTS",
-	"US_SSL_ACCEPT_LATENCY_AVG",
-	"US_WRITABLE_DELAY_AVG",
-	"US_WORST_WRITABLE_DELAY",
-	"US_SSL_RX_DELAY_AVG",
-	"C_PEER_LIMIT_AH_DENIED",
-	"C_PEER_LIMIT_WSI_DENIED",
-	"C_CONNECTIONS_CLIENT",
-	"C_CONNECTIONS_CLIENT_FAILED",
-};
-
-static int
-quantify(struct lws_context *context, int tsi, char *p, int len, int idx,
-	 uint64_t *sum)
-{
-	const lws_humanize_unit_t *schema = humanize_schema_si;
-	struct lws_context_per_thread *pt = &context->pt[tsi];
-	uint64_t u, u1;
-
-	lws_pt_stats_lock(pt);
-	u = pt->lws_stats[idx];
-
-	/* it's supposed to be an average? */
-
-	switch (idx) {
-	case LWSSTATS_US_SSL_ACCEPT_LATENCY_AVG:
-		u1 = pt->lws_stats[LWSSTATS_C_SSL_CONNECTIONS_ACCEPTED];
-		if (u1)
-			u = u / u1;
-		break;
-	case LWSSTATS_US_SSL_RX_DELAY_AVG:
-		u1 = pt->lws_stats[LWSSTATS_C_SSL_CONNS_HAD_RX];
-		if (u1)
-			u = u / u1;
-		break;
-	case LWSSTATS_US_WRITABLE_DELAY_AVG:
-		u1 = pt->lws_stats[LWSSTATS_C_WRITEABLE_CB];
-		if (u1)
-			u = u / u1;
-		break;
-	}
-	lws_pt_stats_unlock(pt);
-
-	*sum += u;
-
-	switch (stat_names[idx][0]) {
-	case 'U':
-		schema = humanize_schema_us;
-		break;
-	case 'B':
-		schema = humanize_schema_si_bytes;
-		break;
-	}
-
-	return lws_humanize(p, len, u, schema);
-}
-
-
-void
-lws_stats_log_dump(struct lws_context *context)
-{
-	struct lws_vhost *v = context->vhost_list;
-	uint64_t summary[LWSSTATS_SIZE];
-	char bufline[128], *p, *end = bufline + sizeof(bufline) - 1;
-	int n, m;
-
-	if (!context->updated)
-		return;
-
-	context->updated = 0;
-	memset(summary, 0, sizeof(summary));
-
-	lwsl_notice("\n");
-	lwsl_notice("LWS internal statistics dump ----->\n");
-	for (n = 0; n < (int)LWS_ARRAY_SIZE(stat_names); n++) {
-		uint64_t u = 0;
-
-		/* if it's all zeroes, don't report it */
-
-		for (m = 0; m < context->count_threads; m++) {
-			struct lws_context_per_thread *pt = &context->pt[m];
-
-			u |= pt->lws_stats[n];
-		}
-		if (!u)
-			continue;
-
-		p = bufline;
-		p += lws_snprintf(p, lws_ptr_diff(end, p), "%28s: ",
-				  stat_names[n]);
-
-		for (m = 0; m < context->count_threads; m++)
-			quantify(context, m, p, lws_ptr_diff(end, p), n, &summary[n]);
-
-		lwsl_notice("%s\n", bufline);
-	}
-
-	lwsl_notice("Simultaneous SSL restriction:  %8d/%d\n",
-			context->simultaneous_ssl,
-			context->simultaneous_ssl_restriction);
-
-	while (v) {
-		if (v->lserv_wsi &&
-		    v->lserv_wsi->position_in_fds_table != LWS_NO_FDS_POS) {
-
-			struct lws_context_per_thread *pt =
-					&context->pt[(int)v->lserv_wsi->tsi];
-			struct lws_pollfd *pfd;
-
-			pfd = &pt->fds[v->lserv_wsi->position_in_fds_table];
-
-			lwsl_notice("  Listen port %d actual POLLIN:   %d\n",
-				    v->listen_port,
-				    (int)pfd->events & LWS_POLLIN);
-		}
-
-		v = v->vhost_next;
-	}
-
-	for (n = 0; n < context->count_threads; n++) {
-		struct lws_context_per_thread *pt = &context->pt[n];
-		struct lws *wl;
-		int m = 0;
-
-		lwsl_notice("PT %d\n", n + 1);
-
-		lws_pt_lock(pt, __func__);
-
-		lwsl_notice("  AH in use / max:                  %d / %d\n",
-				pt->http.ah_count_in_use,
-				context->max_http_header_pool);
-
-		wl = pt->http.ah_wait_list;
-		while (wl) {
-			m++;
-			wl = wl->http.ah_wait_list;
-		}
-
-		lwsl_notice("  AH wait list count / actual:      %d / %d\n",
-				pt->http.ah_wait_list_length, m);
-
-		lws_pt_unlock(pt);
-	}
-
-#if defined(LWS_WITH_PEER_LIMITS)
-	m = 0;
-	for (n = 0; n < (int)context->pl_hash_elements; n++) {
-		lws_start_foreach_llp(struct lws_peer **, peer,
-				      context->pl_hash_table[n]) {
-			m++;
-		} lws_end_foreach_llp(peer, next);
-	}
-
-	lwsl_notice(" Peers: total active %d\n", m);
-	if (m > 10) {
-		m = 10;
-		lwsl_notice("  (showing 10 peers only)\n");
-	}
-
-	if (m) {
-		for (n = 0; n < (int)context->pl_hash_elements; n++) {
-			char buf[72];
-
-			lws_start_foreach_llp(struct lws_peer **, peer,
-					      context->pl_hash_table[n]) {
-				struct lws_peer *df = *peer;
-
-				if (!lws_plat_inet_ntop(df->af, df->addr, buf,
-							sizeof(buf) - 1))
-					strcpy(buf, "unknown");
-#if defined(LWS_ROLE_H1) || defined(LWS_ROLE_H2)
-				lwsl_notice("  peer %s: count wsi: %d, count ah: %d\n",
-					    buf, df->count_wsi,
-					    df->http.count_ah);
-#else
-				lwsl_notice("  peer %s: count wsi: %d\n",
-					    buf, df->count_wsi);
-#endif
-
-				if (!--m)
-					break;
-			} lws_end_foreach_llp(peer, next);
-		}
-	}
-#endif
-
-	lwsl_notice("\n");
-}
-
-void
-lws_stats_bump(struct lws_context_per_thread *pt, int i, uint64_t bump)
-{
-	lws_pt_stats_lock(pt);
-	pt->lws_stats[i] += bump;
-	if (i != LWSSTATS_C_SERVICE_ENTRY) {
-		pt->updated = 1;
-		pt->context->updated = 1;
-	}
-	lws_pt_stats_unlock(pt);
-}
-
-void
-lws_stats_max(struct lws_context_per_thread *pt, int index, uint64_t val)
-{
-	lws_pt_stats_lock(pt);
-	if (val > pt->lws_stats[index]) {
-		pt->lws_stats[index] = val;
-		pt->updated = 1;
-		pt->context->updated = 1;
-	}
-	lws_pt_stats_unlock(pt);
-}
-
-#endif
-
-

lib/core-net/vhost.c

@@ -1,7 +1,7 @@
 /*
  * libwebsockets - small server side websockets and web server implementation
  *
- * Copyright (C) 2010 - 2019 Andy Green <andy@warmcat.com>
+ * Copyright (C) 2010 - 2021 Andy Green <andy@warmcat.com>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to
@@ -119,9 +119,13 @@ lws_role_call_alpn_negotiated(struct lws *wsi, const char *alpn)
 
 	LWS_FOR_EVERY_AVAILABLE_ROLE_START(ar)
 		if (ar->alpn && !strcmp(ar->alpn, alpn) &&
-		    lws_rops_fidx(ar, LWS_ROPS_alpn_negotiated))
+		    lws_rops_fidx(ar, LWS_ROPS_alpn_negotiated)) {
+#if defined(LWS_WITH_SERVER)
+			lws_metrics_tag_wsi_add(wsi, "upg", ar->name);
+#endif
 			return (lws_rops_func_fidx(ar, LWS_ROPS_alpn_negotiated)).
 						   alpn_negotiated(wsi, alpn);
+		}
 	LWS_FOR_EVERY_AVAILABLE_ROLE_END;
 #endif
 	return 0;
@@ -305,6 +309,8 @@ lws_vhd_find_by_pvo(struct lws_context *cx, const char *protname,
 	vh = cx->vhost_list;
 	while (vh) {
 
+		if (vh->protocol_vh_privs) {
+
 		for (n = 0; n < vh->count_protocols; n++) {
 			const struct lws_protocol_vhost_options *pv;
 
@@ -313,7 +319,11 @@ lws_vhd_find_by_pvo(struct lws_context *cx, const char *protname,
 
 			/* this vh has an instance of the required protocol */
 
-			pv = lws_pvo_search(vh->pvo, pvo_name);
+			pv = lws_pvo_search(vh->pvo, protname);
+			if (!pv)
+				continue;
+
+			pv = lws_pvo_search(pv->options, pvo_name);
 			if (!pv)
 				continue;
 
@@ -326,6 +336,8 @@ lws_vhd_find_by_pvo(struct lws_context *cx, const char *protname,
 				 */
 				return vh->protocol_vh_privs[n];
 		}
+		} else
+			lwsl_notice("%s: no privs yet on %s\n", __func__, lws_vh_tag(vh));
 		vh = vh->vhost_next;
 	}
 
@@ -460,7 +472,7 @@ lws_protocol_init(struct lws_context *context)
 
 	context->doing_protocol_init = 1;
 
-	lwsl_notice("%s\n", __func__);
+	lwsl_info("%s\n", __func__);
 
 	while (vh) {
 
@@ -538,10 +550,7 @@ lws_create_vhost(struct lws_context *context,
 	struct lws_protocols *lwsp;
 	int m, f = !info->pvo, fx = 0, abs_pcol_count = 0, sec_pcol_count = 0;
 	char buf[96];
-#if defined(LWS_CLIENT_HTTP_PROXYING) && defined(LWS_WITH_CLIENT) \
-	&& defined(LWS_HAVE_GETENV)
 	char *p;
-#endif
 #if defined(LWS_WITH_SYS_ASYNC_DNS)
 	extern struct lws_protocols lws_async_dns_protocol;
 #endif
@@ -572,6 +581,19 @@ lws_create_vhost(struct lws_context *context,
 		vh->name = "default";
 	else
 		vh->name = info->vhost_name;
+	{
+		char *end = buf + sizeof(buf) - 1;
+		p = buf;
+
+		p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), "%s", vh->name);
+		if (info->iface)
+			p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), "|%s", info->iface);
+		if (info->port && !(info->port & 0xffff))
+			p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), "|%u", info->port);
+	}
+
+	__lws_lc_tag(&context->lcg[LWSLCG_VHOST], &vh->lc, "%s|%s|%d", buf,
+			info->iface ? info->iface : "", info->port);
 
 #if defined(LWS_WITH_SYS_FAULT_INJECTION)
 	vh->fi.name = "vh";
@@ -584,9 +606,6 @@ lws_create_vhost(struct lws_context *context,
 		lws_fi_import(&vh->fi, info->fi);
 #endif
 
-	__lws_lc_tag(&context->lcg[LWSLCG_VHOST], &vh->lc, "%s|%s|%d", vh->name,
-			info->iface ? info->iface : "", info->port);
-
 #if defined(LWS_ROLE_H1) || defined(LWS_ROLE_H2)
 	vh->http.error_document_404 = info->error_document_404;
 #endif
@@ -804,6 +823,23 @@ lws_create_vhost(struct lws_context *context,
 	vh->http.mount_list = info->mounts;
 #endif
 
+#if defined(LWS_WITH_SYS_METRICS) && defined(LWS_WITH_SERVER)
+	{
+		char *end = buf + sizeof(buf) - 1;
+		p = buf;
+
+		p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), "vh.%s", vh->name);
+		if (info->iface)
+			p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), ".%s", info->iface);
+		if (info->port && !(info->port & 0xffff))
+			p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), ".%u", info->port);
+		p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), ".rx");
+		vh->mt_traffic_rx = lws_metric_create(context, 0, buf);
+		p[-2] = 't';
+		vh->mt_traffic_tx = lws_metric_create(context, 0, buf);
+	}
+#endif
+
 #ifdef LWS_WITH_UNIX_SOCK
 	if (LWS_UNIX_SOCK_ENABLED(vh)) {
 		lwsl_info("Creating Vhost '%s' path \"%s\", %d protocols\n",
@@ -909,7 +945,7 @@ lws_create_vhost(struct lws_context *context,
 		goto bail1;
 	}
 #if defined(LWS_WITH_SERVER)
-	lws_context_lock(context, "create_vhost");
+	lws_context_lock(context, __func__);
 	n = _lws_vhost_init_server(info, vh);
 	lws_context_unlock(context);
 	if (n < 0) {
@@ -1397,6 +1433,11 @@ __lws_vhost_destroy2(struct lws_vhost *vh)
 	lws_dll2_foreach_safe(&vh->abstract_instances_owner, NULL, destroy_ais);
 #endif
 
+#if defined(LWS_WITH_SERVER) && defined(LWS_WITH_SYS_METRICS)
+	lws_metric_destroy(&vh->mt_traffic_rx, 0);
+	lws_metric_destroy(&vh->mt_traffic_tx, 0);
+#endif
+
 	lws_dll2_remove(&vh->vh_being_destroyed_list);
 
 #if defined(LWS_WITH_SYS_FAULT_INJECTION)
@@ -1404,6 +1445,7 @@ __lws_vhost_destroy2(struct lws_vhost *vh)
 #endif
 
 	__lws_lc_untag(&vh->lc);
+
 	memset(vh, 0, sizeof(*vh));
 	lws_free(vh);
 }

lib/core-net/wsi-timeout.c

@@ -81,9 +81,6 @@ lws_sul_wsitimeout_cb(lws_sorted_usec_list_t *sul)
 	struct lws *wsi = lws_container_of(sul, struct lws, sul_timeout);
 	struct lws_context_per_thread *pt = &wsi->a.context->pt[(int)wsi->tsi];
 
-	if (wsi->pending_timeout != PENDING_TIMEOUT_USER_OK)
-		lws_stats_bump(pt, LWSSTATS_C_TIMEOUTS, 1);
-
 	/* no need to log normal idle keepalive timeout */
 //		if (wsi->pending_timeout != PENDING_TIMEOUT_HTTP_KEEPALIVE_IDLE)
 #if defined(LWS_ROLE_H1) || defined(LWS_ROLE_H2)

lib/core-net/wsi.c

@@ -342,9 +342,9 @@ lws_rx_flow_control(struct lws *wsi, int _enable)
 
 	/* any bit set in rxflow_bitmap DISABLEs rxflow control */
 	if (en & LWS_RXFLOW_REASON_APPLIES_ENABLE_BIT)
-		wsi->rxflow_bitmap &= (uint8_t)~(en & 0xff);
+		wsi->rxflow_bitmap = (uint8_t)(wsi->rxflow_bitmap & ~(en & 0xff));
 	else
-		wsi->rxflow_bitmap |= (uint8_t)(en & 0xff);
+		wsi->rxflow_bitmap = (uint8_t)(wsi->rxflow_bitmap | (en & 0xff));
 
 	if ((LWS_RXFLOW_PENDING_CHANGE | (!wsi->rxflow_bitmap)) ==
 	    wsi->rxflow_change_to)

lib/core/context.c

@@ -53,20 +53,6 @@ lws_get_library_version(void)
 	return library_version;
 }
 
-#if defined(LWS_WITH_STATS)
-static void
-lws_sul_stats_cb(lws_sorted_usec_list_t *sul)
-{
-	struct lws_context_per_thread *pt = lws_container_of(sul,
-			struct lws_context_per_thread, sul_stats);
-
-	lws_stats_log_dump(pt->context);
-
-	__lws_sul_insert_us(&pt->pt_sul_owner[LWSSULLI_MISS_IF_SUSPENDED],
-			    &pt->sul_stats, 10 * LWS_US_PER_SEC);
-}
-#endif
-
 #if defined(LWS_WITH_NETWORK)
 
 #if defined(LWS_WITH_SYS_STATE)
@@ -379,10 +365,11 @@ lws_create_context(const struct lws_context_creation_info *info)
 	pid_t pid_daemon = get_daemonize_pid();
 #endif
 #if defined(LWS_WITH_NETWORK)
+	const lws_plugin_evlib_t *plev = NULL;
 	unsigned short count_threads = 1;
 	uint8_t *u;
 	uint16_t us_wait_resolution = 0;
-#endif
+
 #if defined(__ANDROID__)
 	struct rlimit rt;
 #endif
@@ -394,9 +381,10 @@ lws_create_context(const struct lws_context_creation_info *info)
 		s1 = 4096,
 #endif
 		size = sizeof(struct lws_context);
+#endif
+
 	int n;
 	unsigned int lpf = info->fd_limit_per_thread;
-	const lws_plugin_evlib_t *plev = NULL;
 #if defined(LWS_WITH_EVLIB_PLUGINS) && defined(LWS_WITH_EVENT_LIBS)
 	struct lws_plugin		*evlib_plugin_list = NULL;
 #if defined(_DEBUG)
@@ -429,10 +417,6 @@ lws_create_context(const struct lws_context_creation_info *info)
 		s = "IPV6-off";
 #endif
 
-#if defined(LWS_WITH_STATS)
-	lwsl_info(" LWS_WITH_STATS        : on\n");
-#endif
-
 	lwsl_notice("%s%s\n", opts_str, s);
 
 	if (lws_plat_context_early_init())
@@ -453,7 +437,6 @@ lws_create_context(const struct lws_context_creation_info *info)
 #if !defined(LWS_PLAT_FREERTOS)
 	size += (count_threads * sizeof(struct lws));
 #endif
-#endif /* network */
 
 #if defined(LWS_WITH_POLL)
 	{
@@ -655,6 +638,10 @@ lws_create_context(const struct lws_context_creation_info *info)
 	context->lcg[LWSLCG_VHOST].tag_prefix = "vh";
 	context->lcg[LWSLCG_WSI_SERVER].tag_prefix = "wsisrv"; /* adopted */
 
+#if defined(LWS_ROLE_H2) || defined(LWS_ROLE_MQTT)
+	context->lcg[LWSLCG_WSI_MUX].tag_prefix = "mux", /* a mux child wsi */
+#endif
+
 #if defined(LWS_WITH_CLIENT)
 	context->lcg[LWSLCG_WSI_CLIENT].tag_prefix = "wsicli";
 #endif
@@ -675,6 +662,77 @@ lws_create_context(const struct lws_context_creation_info *info)
 #endif
 #endif
 
+#if defined(LWS_WITH_SYS_METRICS)
+	/*
+	 * If we're not using secure streams, we can still pass in a linked-
+	 * list of metrics policies
+	 */
+	context->metrics_policies = info->metrics_policies;
+	context->metrics_prefix = info->metrics_prefix;
+
+	context->mt_service = lws_metric_create(context,
+					LWSMTFL_REPORT_DUTY_WALLCLOCK_US |
+					LWSMTFL_REPORT_ONLY_GO, "cpu.svc");
+
+#if defined(LWS_WITH_CLIENT)
+
+	context->mt_conn_dns = lws_metric_create(context,
+						 LWSMTFL_REPORT_MEAN |
+						 LWSMTFL_REPORT_DUTY_WALLCLOCK_US,
+						 "n.cn.dns");
+	context->mt_conn_tcp = lws_metric_create(context,
+						 LWSMTFL_REPORT_MEAN |
+						 LWSMTFL_REPORT_DUTY_WALLCLOCK_US,
+						 "n.cn.tcp");
+	context->mt_conn_tls = lws_metric_create(context,
+						 LWSMTFL_REPORT_MEAN |
+						 LWSMTFL_REPORT_DUTY_WALLCLOCK_US,
+						 "n.cn.tls");
+#if defined(LWS_ROLE_H1) || defined(LWS_ROLE_H2)
+	context->mt_http_txn = lws_metric_create(context,
+						 LWSMTFL_REPORT_MEAN |
+						 LWSMTFL_REPORT_DUTY_WALLCLOCK_US,
+						 "n.http.txn");
+#endif
+
+	context->mth_conn_failures = lws_metric_create(context,
+					LWSMTFL_REPORT_HIST, "n.cn.failures");
+
+#if defined(LWS_WITH_SYS_ASYNC_DNS)
+	context->mt_adns_cache = lws_metric_create(context,
+						   LWSMTFL_REPORT_MEAN |
+						   LWSMTFL_REPORT_DUTY_WALLCLOCK_US,
+						   "n.cn.adns");
+#endif
+#if defined(LWS_WITH_SECURE_STREAMS)
+	context->mth_ss_conn = lws_metric_create(context, LWSMTFL_REPORT_HIST,
+						 "n.ss.conn");
+#endif
+#if defined(LWS_WITH_SECURE_STREAMS_PROXY_API)
+	context->mt_ss_cliprox_conn = lws_metric_create(context,
+			LWSMTFL_REPORT_HIST,
+							"n.ss.cliprox.conn");
+	context->mt_ss_cliprox_paylat = lws_metric_create(context,
+							  LWSMTFL_REPORT_MEAN |
+							  LWSMTFL_REPORT_DUTY_WALLCLOCK_US,
+							  "n.ss.cliprox.paylat");
+	context->mt_ss_proxcli_paylat = lws_metric_create(context,
+							  LWSMTFL_REPORT_MEAN |
+							  LWSMTFL_REPORT_DUTY_WALLCLOCK_US,
+							  "n.ss.proxcli.paylat");
+#endif
+
+#endif /* network + metrics + client */
+
+#if defined(LWS_WITH_SERVER)
+	context->mth_srv = lws_metric_create(context,
+					     LWSMTFL_REPORT_HIST, "n.srv");
+#endif /* network + metrics + server */
+
+#endif /* network + metrics */
+
+#endif /* network */
+
 	/*
 	 * Proxy group
 	 */
@@ -713,11 +771,7 @@ lws_create_context(const struct lws_context_creation_info *info)
 #if defined(LWS_WITH_NETWORK)
 	context->undestroyed_threads = count_threads;
 	context->count_threads = count_threads;
-#if defined(LWS_WITH_DETAILED_LATENCY)
+
-	context->detailed_latency_cb = info->detailed_latency_cb;
-	context->detailed_latency_filepath = info->detailed_latency_filepath;
-	context->latencies_fd = -1;
-#endif
 #if defined(LWS_ROLE_WS) && defined(LWS_WITHOUT_EXTENSIONS)
         if (info->extensions)
                 lwsl_warn("%s: LWS_WITHOUT_EXTENSIONS but extensions ptr set\n", __func__);
@@ -1144,12 +1198,6 @@ lws_create_context(const struct lws_context_creation_info *info)
 #endif
 #endif
 
-#if defined(LWS_WITH_STATS)
-	context->pt[0].sul_stats.cb = lws_sul_stats_cb;
-	__lws_sul_insert_us(&context->pt[0].pt_sul_owner[LWSSULLI_MISS_IF_SUSPENDED],
-			    &context->pt[0].sul_stats, 10 * LWS_US_PER_SEC);
-#endif
-
 #if defined(LWS_HAVE_SYS_CAPABILITY_H) && defined(LWS_HAVE_LIBCAP)
 	memcpy(context->caps, info->caps, sizeof(context->caps));
 	context->count_caps = info->count_caps;
@@ -1364,8 +1412,10 @@ bail_libuv_aware:
 	return NULL;
 #endif
 
+#if defined(LWS_WITH_NETWORK)
 fail_event_libs:
 	lwsl_err("Requested event library support not configured\n");
+#endif
 
 free_context_fail:
 	lws_free(context);
@@ -1598,6 +1648,11 @@ lws_context_destroy(struct lws_context *context)
 		context->being_destroyed = 1;
 
 #if defined(LWS_WITH_NETWORK)
+
+#if defined(LWS_WITH_SYS_METRICS)
+		lws_metrics_dump(context);
+#endif
+
 		/*
 		 * Close any vhost listen wsi
 		 *
@@ -1819,7 +1874,6 @@ next:
 		lwsl_debug("%p: post pdl\n", __func__);
 #endif
 
-		lws_stats_log_dump(context);
 #if defined(LWS_WITH_NETWORK)
 		lws_ssl_context_destroy(context);
 #endif
@@ -1982,6 +2036,10 @@ next:
 		lws_mutex_refcount_destroy(&context->mr);
 #endif
 
+#if defined(LWS_WITH_SYS_METRICS) && defined(LWS_WITH_NETWORK)
+		lws_metrics_destroy(context);
+#endif
+
 		if (context->external_baggage_free_on_destroy)
 			free(context->external_baggage_free_on_destroy);
 

lib/core/logs.c

@@ -55,6 +55,42 @@ __lws_lc_tag(lws_lifecycle_group_t *grp, lws_lifecycle_t *lc,
 	va_list ap;
 	int n = 1;
 
+	if (*lc->gutag == '[') {
+		/* appending inside [] */
+
+		char *cp = strchr(lc->gutag, ']');
+		char rend[96];
+		size_t ll, k;
+		int n;
+
+		if (!cp)
+			return;
+
+		/* length of closing brace and anything else after it */
+		k = strlen(cp);
+
+		/* compute the remaining gutag unused */
+		ll = sizeof(lc->gutag) - lws_ptr_diff_size_t(cp, lc->gutag) - k - 1;
+		if (ll > sizeof(rend) - 1)
+			ll = sizeof(rend) - 1;
+		va_start(ap, format);
+		n = vsnprintf(rend, ll, format, ap);
+		va_end(ap);
+
+		if ((unsigned int)n > ll)
+			n = (int)ll;
+
+		/* shove the trailer up by what we added */
+		memmove(cp + n, cp, k);
+		assert(k + (unsigned int)n < sizeof(lc->gutag));
+		cp[k + (unsigned int)n] = '\0';
+		/* copy what we added into place */
+		memcpy(cp, rend, (unsigned int)n);
+
+		return;
+	}
+
+	assert(grp);
 	assert(grp->tag_prefix); /* lc group must have a tag prefix string */
 
 	lc->gutag[0] = '[';

lib/core/private-lib-core.h

@@ -139,6 +139,41 @@
 
 #include "libwebsockets.h"
 
+/*
+ * lws_dsh
+*/
+
+typedef struct lws_dsh_obj_head {
+	lws_dll2_owner_t		owner;
+	size_t				total_size; /* for this kind in dsh */
+	int				kind;
+} lws_dsh_obj_head_t;
+
+typedef struct lws_dsh_obj {
+	lws_dll2_t			list;	/* must be first */
+	struct lws_dsh	  		*dsh;	/* invalid when on free list */
+	size_t				size;	/* invalid when on free list */
+	size_t				asize;
+	int				kind; /* so we can account at free */
+} lws_dsh_obj_t;
+
+typedef struct lws_dsh {
+	lws_dll2_t			list;
+	uint8_t				*buf;
+	lws_dsh_obj_head_t		*oha;	/* array of object heads/kind */
+	size_t				buffer_size;
+	size_t				locally_in_use;
+	size_t				locally_free;
+	int				count_kinds;
+	uint8_t				being_destroyed;
+	/*
+	 * Overallocations at create:
+	 *
+	 *  - the buffer itself
+	 *  - the object heads array
+	 */
+} lws_dsh_t;
+
  /*
   *
   *  ------ lifecycle defines ------
@@ -285,6 +320,9 @@ struct lws;
 #include "private-lib-system-fault-injection.h"
 #endif
 
+#include "private-lib-system-metrics.h"
+
+
 struct lws_foreign_thread_pollfd {
 	struct lws_foreign_thread_pollfd *next;
 	int fd_index;
@@ -328,6 +366,10 @@ enum {
 
 	LWSLCG_WSI_SERVER,		/* server wsi */
 
+#if defined(LWS_ROLE_H2) || defined(LWS_ROLE_MQTT)
+	LWSLCG_WSI_MUX,			/* a mux child wsi */
+#endif
+
 #if defined(LWS_WITH_CLIENT)
 	LWSLCG_WSI_CLIENT,		/* client wsi */
 #endif
@@ -418,21 +460,53 @@ struct lws_context {
 	struct http2_settings			set;
 #endif
 
-#if defined(LWS_WITH_SERVER_STATUS)
-	struct lws_conn_stats			conn_stats;
-#endif
 #if LWS_MAX_SMP > 1
 	struct lws_mutex_refcount		mr;
 #endif
 
-#if defined(LWS_WITH_NETWORK)
+#if defined(LWS_WITH_SYS_METRICS)
+	lws_dll2_owner_t			owner_mtr_dynpol;
+	/**< owner for lws_metric_policy_dyn_t (dynamic part of metric pols) */
+	lws_dll2_owner_t			owner_mtr_no_pol;
+	/**< owner for lws_metric_pub_t with no policy to bind to */
+#endif
 
+#if defined(LWS_WITH_NETWORK)
 /*
  * LWS_WITH_NETWORK =====>
  */
 
 	lws_dll2_owner_t		owner_vh_being_destroyed;
 
+	lws_metric_t			*mt_service; /* doing service */
+	const lws_metric_policy_t	*metrics_policies;
+	const char			*metrics_prefix;
+
+#if defined(LWS_WITH_SYS_METRICS) && defined(LWS_WITH_CLIENT)
+	lws_metric_t			*mt_conn_tcp; /* client tcp conns */
+	lws_metric_t			*mt_conn_tls; /* client tcp conns */
+	lws_metric_t			*mt_conn_dns; /* client dns external lookups */
+	lws_metric_t			*mth_conn_failures; /* histogram of conn failure reasons */
+#if defined(LWS_ROLE_H1) || defined(LWS_ROLE_H2)
+	lws_metric_t			*mt_http_txn; /* client http transaction */
+#endif
+#if defined(LWS_WITH_SYS_ASYNC_DNS)
+	lws_metric_t			*mt_adns_cache; /* async dns lookup lat */
+#endif
+#if defined(LWS_WITH_SECURE_STREAMS)
+	lws_metric_t			*mth_ss_conn; /* SS connection outcomes */
+#endif
+#if defined(LWS_WITH_SECURE_STREAMS_PROXY_API)
+	lws_metric_t			*mt_ss_cliprox_conn; /* SS cli->prox conn */
+	lws_metric_t			*mt_ss_cliprox_paylat; /* cli->prox payload latency */
+	lws_metric_t			*mt_ss_proxcli_paylat; /* prox->cli payload latency */
+#endif
+#endif /* client */
+
+#if defined(LWS_WITH_SERVER)
+	lws_metric_t			*mth_srv;
+#endif
+
 #if defined(LWS_WITH_EVENT_LIBS)
 	struct lws_plugin		*evlib_plugin_list;
 	void				*evlib_ctx; /* overallocated */
@@ -498,9 +572,6 @@ struct lws_context {
 	const struct lws_tls_ops	*tls_ops;
 #endif
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	det_lat_buf_cb_t		detailed_latency_cb;
-#endif
 #if defined(LWS_WITH_PLUGINS)
 	struct lws_plugin		*plugin_list;
 #endif
@@ -531,9 +602,6 @@ struct lws_context {
 #if !defined(LWS_PLAT_FREERTOS)
 	const char *username, *groupname;
 #endif
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	const char *detailed_latency_filepath;
-#endif
 
 #if defined(LWS_AMAZON_RTOS) && defined(LWS_WITH_MBEDTLS)
 	mbedtls_entropy_context mec;
@@ -589,6 +657,9 @@ struct lws_context {
 	uint64_t options;
 
 	time_t last_ws_ping_pong_check_s;
+#if defined(LWS_WITH_SECURE_STREAMS)
+	time_t					last_policy;
+#endif
 
 #if defined(LWS_PLAT_FREERTOS)
 	unsigned long time_last_state_dump;
@@ -607,9 +678,6 @@ struct lws_context {
 	int count_cgi_spawned;
 #endif
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	int latencies_fd;
-#endif
 	unsigned int fd_limit_per_thread;
 	unsigned int timeout_secs;
 	unsigned int pt_serv_buf_size;
@@ -667,10 +735,6 @@ struct lws_context {
 	uint8_t captive_portal_detect_type;
 
 	uint8_t		destroy_state; /* enum lws_context_destroy */
-
-#if defined(LWS_WITH_STATS)
-	uint8_t updated;
-#endif
 };
 
 #define lws_get_context_protocol(ctx, x) ctx->vhost_list->protocols[x]

lib/plat/freertos/freertos-service.c

@@ -50,7 +50,6 @@ _lws_plat_service_tsi(struct lws_context *context, int timeout_ms, int tsi)
 		return 1;
 
 	pt = &context->pt[tsi];
-	lws_stats_bump(pt, LWSSTATS_C_SERVICE_ENTRY, 1);
 
 	{
 		unsigned long m = lws_now_secs();
@@ -142,15 +141,6 @@ again:
 			n = select(max_fd + 1, &readfds, &writefds, &errfds, ptv);
 			n = 0;
 
-	#if defined(LWS_WITH_DETAILED_LATENCY)
-			/*
-			 * so we can track how long it took before we actually read a POLLIN
-			 * that was signalled when we last exited poll()
-			 */
-			if (context->detailed_latency_cb)
-				pt->ust_left_poll = lws_now_usecs();
-	#endif
-
 			for (m = 0; m < (int)pt->fds_count; m++) {
 				c = 0;
 				if (FD_ISSET(pt->fds[m].fd, &readfds)) {

lib/plat/unix/unix-service.c

@@ -71,6 +71,9 @@ _lws_plat_service_tsi(struct lws_context *context, int timeout_ms, int tsi)
 	volatile struct lws_context_per_thread *vpt;
 	struct lws_context_per_thread *pt;
 	lws_usec_t timeout_us, us;
+#if defined(LWS_WITH_SYS_METRICS)
+	lws_usec_t a, b;
+#endif
 	int n;
 #if (defined(LWS_ROLE_WS) && !defined(LWS_WITHOUT_EXTENSIONS)) || defined(LWS_WITH_TLS)
 	int m;
@@ -81,11 +84,14 @@ _lws_plat_service_tsi(struct lws_context *context, int timeout_ms, int tsi)
 	if (!context)
 		return 1;
 
+#if defined(LWS_WITH_SYS_METRICS)
+	b =
+#endif
+			us = lws_now_usecs();
+
 	pt = &context->pt[tsi];
 	vpt = (volatile struct lws_context_per_thread *)pt;
 
-	lws_stats_bump(pt, LWSSTATS_C_SERVICE_ENTRY, 1);
-
 	if (timeout_ms < 0)
 		timeout_ms = 0;
 	else
@@ -108,7 +114,6 @@ _lws_plat_service_tsi(struct lws_context *context, int timeout_ms, int tsi)
 		pt->service_tid_detected = 1;
 	}
 
-	us = lws_now_usecs();
 	lws_pt_lock(pt, __func__);
 	/*
 	 * service ripe scheduled events, and limit wait to next expected one
@@ -144,15 +149,9 @@ _lws_plat_service_tsi(struct lws_context *context, int timeout_ms, int tsi)
 	vpt->inside_poll = 0;
 	lws_memory_barrier();
 
-	#if defined(LWS_WITH_DETAILED_LATENCY)
+#if defined(LWS_WITH_SYS_METRICS)
-	/*
+	b = lws_now_usecs();
-	 * so we can track how long it took before we actually read a
-	 * POLLIN that was signalled when we last exited poll()
-	 */
-	if (context->detailed_latency_cb)
-		pt->ust_left_poll = lws_now_usecs();
 #endif
-
 	/* Collision will be rare and brief.  Spin until it completes */
 	while (vpt->foreign_spinlock)
 		;
@@ -207,14 +206,16 @@ _lws_plat_service_tsi(struct lws_context *context, int timeout_ms, int tsi)
 #if (defined(LWS_ROLE_WS) && !defined(LWS_WITHOUT_EXTENSIONS)) || defined(LWS_WITH_TLS)
 		!m &&
 #endif
-		!n) { /* nothing to do */
+		!n) /* nothing to do */
 		lws_service_do_ripe_rxflow(pt);
+	else
+		if (_lws_plat_service_forced_tsi(context, tsi) < 0)
+			return -1;
 
-		return 0;
+#if defined(LWS_WITH_SYS_METRICS)
-	}
+	lws_metric_event(context->mt_service, METRES_GO,
-
+			 (u_mt_t) (a + (lws_now_usecs() - b)));
-	if (_lws_plat_service_forced_tsi(context, tsi) < 0)
+#endif
-		return -1;
 
 	if (pt->destroy_self) {
 		lws_context_destroy(pt->context);

lib/roles/h1/ops-h1.c

@@ -521,19 +521,6 @@ try_pollout:
 			return LWS_HPI_RET_HANDLED;
 		}
 
-		lws_stats_bump(pt, LWSSTATS_C_WRITEABLE_CB, 1);
-#if defined(LWS_WITH_STATS)
-		if (wsi->active_writable_req_us) {
-			uint64_t ul = lws_now_usecs() -
-					wsi->active_writable_req_us;
-
-			lws_stats_bump(pt, LWSSTATS_US_WRITABLE_DELAY_AVG, ul);
-			lws_stats_max(pt,
-				  LWSSTATS_US_WORST_WRITABLE_DELAY, ul);
-			wsi->active_writable_req_us = 0;
-		}
-#endif
-
 		n = user_callback_handle_rxflow(wsi->a.protocol->callback, wsi,
 						LWS_CALLBACK_HTTP_WRITEABLE,
 						wsi->user_space, NULL, 0);
@@ -934,6 +921,7 @@ rops_adoption_bind_h1(struct lws *wsi, int type, const char *vh_prot_name)
 #if defined(LWS_WITH_HTTP2)
 	if ((!(type & LWS_ADOPT_ALLOW_SSL)) && (wsi->a.vhost->options & LWS_SERVER_OPTION_H2_PRIOR_KNOWLEDGE)) {
 		lwsl_info("http/2 prior knowledge\n");
+		lws_metrics_tag_wsi_add(wsi, "upg", "h2_prior");
 		lws_role_call_alpn_negotiated(wsi, "h2");
 	}
 	else

lib/roles/h2/hpack.c

@@ -1422,7 +1422,7 @@ int lws_add_http2_header_by_name(struct lws *wsi, const unsigned char *name,
 
 	*((*p)++) = 0; /* literal hdr, literal name,  */
 
-	*((*p)++) = 0 | (uint8_t)lws_h2_num_start(7, (unsigned long)len); /* non-HUF */
+	*((*p)++) = (uint8_t)(0 | (uint8_t)lws_h2_num_start(7, (unsigned long)len)); /* non-HUF */
 	if (lws_h2_num(7, (unsigned long)len, p, end))
 		return 1;
 
@@ -1432,7 +1432,7 @@ int lws_add_http2_header_by_name(struct lws *wsi, const unsigned char *name,
 	while(len--)
 		*((*p)++) = (uint8_t)tolower((int)*name++);
 
-	*((*p)++) = 0 | (uint8_t)lws_h2_num_start(7, (unsigned long)length); /* non-HUF */
+	*((*p)++) = (uint8_t)(0 | (uint8_t)lws_h2_num_start(7, (unsigned long)length)); /* non-HUF */
 	if (lws_h2_num(7, (unsigned long)length, p, end))
 		return 1;
 

lib/roles/h2/http2.c

@@ -261,6 +261,11 @@ lws_wsi_server_new(struct lws_vhost *vh, struct lws *parent_wsi,
 		return NULL;
 	}
 
+#if defined(LWS_WITH_SERVER)
+	if (lwsi_role_server(parent_wsi))
+		lws_metrics_caliper_bind(wsi->cal_conn, wsi->a.context->mth_srv);
+#endif
+
 	h2n->highest_sid_opened = sid;
 
 	lws_wsi_mux_insert(wsi, parent_wsi, sid);
@@ -281,10 +286,6 @@ lws_wsi_server_new(struct lws_vhost *vh, struct lws *parent_wsi,
 	if (lws_ensure_user_space(wsi))
 		goto bail1;
 
-#if defined(LWS_WITH_SERVER_STATUS)
-	wsi->a.vhost->conn_stats.h2_subs++;
-#endif
-
 #if defined(LWS_WITH_SERVER) && defined(LWS_WITH_SECURE_STREAMS)
 	if (lws_adopt_ss_server_accept(wsi))
 		goto bail1;
@@ -362,10 +363,6 @@ lws_wsi_h2_adopt(struct lws *parent_wsi, struct lws *wsi)
 
 	lws_callback_on_writable(wsi);
 
-#if defined(LWS_WITH_SERVER_STATUS)
-	wsi->a.vhost->conn_stats.h2_subs++;
-#endif
-
 	return wsi;
 
 bail1:
@@ -818,9 +815,6 @@ int lws_h2_do_pps_send(struct lws *wsi)
 			h2n->swsi->h2.END_STREAM = 1;
 			lwsl_info("servicing initial http request\n");
 
-#if defined(LWS_WITH_SERVER_STATUS)
-			wsi->a.vhost->conn_stats.h2_trans++;
-#endif
 #if defined(LWS_WITH_SERVER)
 			if (lws_http_action(h2n->swsi))
 				goto bail;
@@ -1713,9 +1707,6 @@ lws_h2_parse_end_of_frame(struct lws *wsi)
 		lws_http_compression_validate(h2n->swsi);
 #endif
 
-#if defined(LWS_WITH_SERVER_STATUS)
-		wsi->a.vhost->conn_stats.h2_trans++;
-#endif
 		p = lws_hdr_simple_ptr(h2n->swsi, WSI_TOKEN_HTTP_COLON_METHOD);
 		/*
 		 * duplicate :path into the individual method uri header
@@ -2532,10 +2523,6 @@ lws_h2_client_handshake(struct lws *wsi)
 	if (lws_finalize_http_header(wsi, &p, end))
 		goto fail_length;
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	wsi->detlat.earliest_write_req_pre_write = lws_now_usecs();
-#endif
-
 	m = LWS_WRITE_HTTP_HEADERS;
 #if defined(LWS_WITH_CLIENT)
 	/* below is not needed in spec, indeed it destroys the long poll

lib/roles/h2/ops-h2.c

@@ -521,13 +521,12 @@ rops_check_upgrades_h2(struct lws *wsi)
 	if (!p || strcmp(p, "websocket"))
 		return LWS_UPG_RET_CONTINUE;
 
-#if defined(LWS_WITH_SERVER_STATUS)
-	wsi->a.vhost->conn_stats.ws_upg++;
-#endif
 	lwsl_info("Upgrade h2 to ws\n");
 	lws_mux_mark_immortal(wsi);
 	wsi->h2_stream_carries_ws = 1;
 
+	lws_metrics_tag_wsi_add(wsi, "upg", "ws_over_h2");
+
 	if (lws_process_ws_upgrade(wsi))
 		return LWS_UPG_RET_BAIL;
 
@@ -1254,9 +1253,6 @@ rops_alpn_negotiated_h2(struct lws *wsi, const char *alpn)
 #endif
 
 	wsi->upgraded_to_http2 = 1;
-#if defined(LWS_WITH_SERVER_STATUS)
-	wsi->a.vhost->conn_stats.h2_alpn++;
-#endif
 
 	/* adopt the header info */
 

lib/roles/http/client/client-http.c

@@ -67,7 +67,10 @@ lws_http_client_socket_service(struct lws *wsi, struct lws_pollfd *pollfd)
 		 * timeout protection set in client-handshake.c
 		 */
 		if (pollfd->revents & LWS_POLLOUT)
-			lws_client_connect_3_connect(wsi, NULL, NULL, 0, NULL);
+			if (lws_client_connect_3_connect(wsi, NULL, NULL, 0, NULL) == NULL) {
+				lwsl_client("closed\n");
+				return -1;
+			}
 		break;
 
 #if defined(LWS_WITH_SOCKS5)
@@ -203,16 +206,6 @@ start_ws_handshake:
 			}
 		}
 #endif
-#if defined(LWS_WITH_DETAILED_LATENCY)
-		if (context->detailed_latency_cb) {
-			wsi->detlat.type = LDLT_TLS_NEG_CLIENT;
-			wsi->detlat.latencies[LAT_DUR_PROXY_CLIENT_REQ_TO_WRITE] =
-				(uint32_t)(lws_now_usecs() -
-				wsi->detlat.earliest_write_req_pre_write);
-			wsi->detlat.latencies[LAT_DUR_USERCB] = 0;
-			lws_det_lat_cb(wsi->a.context, &wsi->detlat);
-		}
-#endif
 
 #if defined (LWS_WITH_HTTP2)
 		if (wsi->client_h2_alpn && lwsi_state(wsi) != LRS_H1C_ISSUE_HANDSHAKE2) {
@@ -271,9 +264,7 @@ hs2:
 			  "(wsistate 0x%lx), w sock %d\n",
 			  __func__, lws_wsi_tag(wsi),
 			  (unsigned long)wsi->wsistate, wsi->desc.sockfd);
-#if defined(LWS_WITH_DETAILED_LATENCY)
+
-		wsi->detlat.earliest_write_req_pre_write = lws_now_usecs();
-#endif
 		n = lws_ssl_capable_write(wsi, (unsigned char *)sb, lws_ptr_diff_size_t(p, sb));
 		switch (n) {
 		case LWS_SSL_CAPABLE_ERROR:
@@ -485,6 +476,10 @@ lws_http_transaction_completed_client(struct lws *wsi)
 	lwsl_info("%s: %s (%s)\n", __func__, lws_wsi_tag(wsi),
 			wsi->a.protocol->name);
 
+	// if (wsi->http.ah && wsi->http.ah->http_response)
+	/* we're only judging if any (200, or 500 etc) http txn completed */
+	lws_metrics_caliper_report(wsi->cal_conn, METRES_GO);
+
 	if (user_callback_handle_rxflow(wsi->a.protocol->callback, wsi,
 					LWS_CALLBACK_COMPLETED_CLIENT_HTTP,
 					wsi->user_space, NULL, 0)) {
@@ -1210,6 +1205,8 @@ lws_generate_client_handshake(struct lws *wsi, char *pkt)
 	if (wsi->client_http_body_pending)
 		lws_callback_on_writable(wsi);
 
+	lws_metrics_caliper_bind(wsi->cal_conn, wsi->a.context->mt_http_txn);
+
 	// puts(pkt);
 
 	return p;

lib/roles/http/header.c

@@ -125,9 +125,6 @@ lws_finalize_write_http_header(struct lws *wsi, unsigned char *start,
 	p = *pp;
 	len = lws_ptr_diff(p, start);
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	wsi->detlat.earliest_write_req_pre_write = lws_now_usecs();
-#endif
 	if (lws_write(wsi, start, (unsigned int)len, LWS_WRITE_HTTP_HEADERS) != len)
 		return 1;
 
@@ -316,6 +313,7 @@ lws_add_http_header_status(struct lws *wsi, unsigned int _code,
 	unsigned char code_and_desc[60];
 	int n;
 
+	wsi->http.response_code = code;
 #ifdef LWS_WITH_ACCESS_LOG
 	wsi->http.access_log.response = (int)code;
 #endif
@@ -482,9 +480,6 @@ lws_return_http_status(struct lws *wsi, unsigned int code,
 		 *
 		 * Solve it by writing the headers now...
 		 */
-#if defined(LWS_WITH_DETAILED_LATENCY)
-		wsi->detlat.earliest_write_req_pre_write = lws_now_usecs();
-#endif
 		m = lws_write(wsi, start, lws_ptr_diff_size_t(p, start),
 			      LWS_WRITE_HTTP_HEADERS);
 		if (m != lws_ptr_diff(p, start))

lib/roles/http/parsers.c

@@ -236,11 +236,8 @@ lws_header_table_attach(struct lws *wsi, int autoservice)
 
 	n = pt->http.ah_count_in_use == (int)context->max_http_header_pool;
 #if defined(LWS_WITH_PEER_LIMITS)
-	if (!n) {
+	if (!n)
 		n = lws_peer_confirm_ah_attach_ok(context, wsi->peer);
-		if (n)
-			lws_stats_bump(pt, LWSSTATS_C_PEER_LIMIT_AH_DENIED, 1);
-	}
 #endif
 	if (n) {
 		/*
@@ -375,12 +372,7 @@ int __lws_header_table_detach(struct lws *wsi, int autoservice)
 			wsi = *pwsi;
 			pwsi_eligible = pwsi;
 		}
-#if defined(LWS_WITH_PEER_LIMITS)
+
-		else
-			if (!(*pwsi)->http.ah_wait_list)
-				lws_stats_bump(pt,
-					LWSSTATS_C_PEER_LIMIT_AH_DENIED, 1);
-#endif
 		pwsi = &(*pwsi)->http.ah_wait_list;
 	}
 

lib/roles/http/private-lib-roles-http.h

@@ -248,6 +248,9 @@ struct _lws_http_mode_related {
 #ifdef LWS_WITH_ACCESS_LOG
 	struct lws_access_log access_log;
 #endif
+#if defined(LWS_WITH_SERVER)
+	unsigned int response_code;
+#endif
 #ifdef LWS_WITH_CGI
 	struct lws_cgi *cgi; /* wsi being cgi stream have one of these */
 #endif

lib/roles/http/server/server.c

@@ -797,6 +797,10 @@ lws_find_mount(struct lws *wsi, const char *uri_ptr, int uri_len)
 		     uri_ptr[hm->mountpoint_len] == '/' ||
 		     hm->mountpoint_len == 1)
 		    ) {
+#if defined(LWS_WITH_SYS_METRICS)
+			lws_metrics_tag_wsi_add(wsi, "mnt", hm->mountpoint);
+#endif
+
 			if (hm->origin_protocol == LWSMPRO_CALLBACK ||
 			    ((hm->origin_protocol == LWSMPRO_CGI ||
 			     lws_hdr_total_length(wsi, WSI_TOKEN_GET_URI) ||
@@ -1416,6 +1420,9 @@ lws_http_action(struct lws *wsi)
 	if (meth < 0 || meth >= (int)LWS_ARRAY_SIZE(method_names))
 		goto bail_nuke_ah;
 
+	lws_metrics_tag_wsi_add(wsi, "vh", wsi->a.vhost->name);
+	lws_metrics_tag_wsi_add(wsi, "meth", method_names[meth]);
+
 	/* we insist on absolute paths */
 
 	if (!uri_ptr || uri_ptr[0] != '/') {
@@ -2076,17 +2083,9 @@ raw_transition:
 		} else
 			lwsl_info("no host\n");
 
-		if (!lwsi_role_h2(wsi) || !lwsi_role_server(wsi)) {
+		if ((!lwsi_role_h2(wsi) || !lwsi_role_server(wsi)) &&
-#if defined(LWS_WITH_SERVER_STATUS)
+		    (!wsi->conn_stat_done))
-			wsi->a.vhost->conn_stats.h1_trans++;
+			wsi->conn_stat_done = 1;
-#endif
-			if (!wsi->conn_stat_done) {
-#if defined(LWS_WITH_SERVER_STATUS)
-				wsi->a.vhost->conn_stats.h1_conn++;
-#endif
-				wsi->conn_stat_done = 1;
-			}
-		}
 
 		/* check for unwelcome guests */
 #if defined(LWS_WITH_HTTP_UNCOMMON_HEADERS)
@@ -2121,9 +2120,6 @@ raw_transition:
 							uri_ptr, uri_len, meth);
 
 					/* wsi close will do the log */
-#endif
-#if defined(LWS_WITH_SERVER_STATUS)
-					wsi->a.vhost->conn_stats.rejected++;
 #endif
 					/*
 					 * We don't want anything from
@@ -2215,18 +2211,14 @@ raw_transition:
 
 			if (!strcasecmp(up, "websocket")) {
 #if defined(LWS_ROLE_WS)
-#if defined(LWS_WITH_SERVER_STATUS)
+				lws_metrics_tag_wsi_add(wsi, "upg", "ws");
-				wsi->a.vhost->conn_stats.ws_upg++;
-#endif
 				lwsl_info("Upgrade to ws\n");
 				goto upgrade_ws;
 #endif
 			}
 #if defined(LWS_WITH_HTTP2)
 			if (!strcasecmp(up, "h2c")) {
-#if defined(LWS_WITH_SERVER_STATUS)
+				lws_metrics_tag_wsi_add(wsi, "upg", "h2c");
-				wsi->a.vhost->conn_stats.h2_upg++;
-#endif
 				lwsl_info("Upgrade to h2c\n");
 				goto upgrade_h2c;
 			}
@@ -2379,6 +2371,15 @@ lws_http_transaction_completed(struct lws *wsi)
 		return 0;
 	}
 
+#if defined(LWS_WITH_SYS_METRICS)
+	{
+		char tmp[10];
+
+		lws_snprintf(tmp, sizeof(tmp), "%u", wsi->http.response_code);
+		lws_metrics_tag_wsi_add(wsi, "status", tmp);
+	}
+#endif
+
 	lwsl_info("%s: %s\n", __func__, lws_wsi_tag(wsi));
 
 #if defined(LWS_WITH_HTTP_STREAM_COMPRESSION)

lib/roles/mqtt/client/client-mqtt.c

@@ -264,17 +264,6 @@ lws_mqtt_client_socket_service(struct lws *wsi, struct lws_pollfd *pollfd,
 			wsi->tls.ssl = NULL;
 #endif /* LWS_WITH_TLS */
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-		if (context->detailed_latency_cb) {
-			wsi->detlat.type = LDLT_TLS_NEG_CLIENT;
-			wsi->detlat.latencies[LAT_DUR_PROXY_CLIENT_REQ_TO_WRITE] =
-				(uint32_t)(lws_now_usecs() -
-				wsi->detlat.earliest_write_req_pre_write);
-			wsi->detlat.latencies[LAT_DUR_USERCB] = 0;
-			lws_det_lat_cb(wsi->a.context, &wsi->detlat);
-		}
-#endif
-
 		/* fallthru */
 
 #if defined(LWS_WITH_SOCKS5)

lib/roles/mqtt/mqtt.c

@@ -992,7 +992,7 @@ cmd_completion:
 				lws_set_timeout(wsi, 0, 0);
 
 				w = lws_create_new_server_wsi(wsi->a.vhost,
-							      wsi->tsi, "mqtt");
+							      wsi->tsi, "mqtt_sid1");
 				if (!w) {
 					lwsl_notice("%s: sid 1 migrate failed\n",
 							__func__);
@@ -1042,10 +1042,6 @@ cmd_completion:
 						__func__, lws_wsi_tag(wsi),
 						lws_wsi_tag(w));
 
-			#if defined(LWS_WITH_SERVER_STATUS)
-				wsi->a.vhost->conn_stats.h2_subs++;
-			#endif
-
 				/*
 				 * It was the last thing we were waiting for
 				 * before we can be fully ESTABLISHED
@@ -2107,10 +2103,6 @@ lws_wsi_mqtt_adopt(struct lws *parent_wsi, struct lws *wsi)
 	lws_mqtt_set_client_established(wsi);
 	lws_callback_on_writable(wsi);
 
-#if defined(LWS_WITH_SERVER_STATUS)
-	wsi->a.vhost->conn_stats.mqtt_subs++;
-#endif
-
 	return wsi;
 
 bail1:

lib/roles/raw-skt/ops-raw-skt.c

@@ -120,6 +120,8 @@ rops_handle_POLLIN_raw_skt(struct lws_context_per_thread *pt, struct lws *wsi,
 			buffered = lws_buflist_aware_read(pt, wsi, &ebuf, 1, __func__);
 			switch (ebuf.len) {
 			case 0:
+				if (wsi->unix_skt)
+					break;
 				lwsl_info("%s: read 0 len\n", __func__);
 				wsi->seen_zero_length_recv = 1;
 				if (lws_change_pollfd(wsi, LWS_POLLIN, 0))
@@ -202,18 +204,6 @@ try_pollout:
 	/* clear back-to-back write detection */
 	wsi->could_have_pending = 0;
 
-	lws_stats_bump(pt, LWSSTATS_C_WRITEABLE_CB, 1);
-#if defined(LWS_WITH_STATS)
-	if (wsi->active_writable_req_us) {
-		uint64_t ul = lws_now_usecs() -
-				wsi->active_writable_req_us;
-
-		lws_stats_bump(pt, LWSSTATS_US_WRITABLE_DELAY_AVG, ul);
-		lws_stats_max(pt,
-			  LWSSTATS_US_WORST_WRITABLE_DELAY, ul);
-		wsi->active_writable_req_us = 0;
-	}
-#endif
 	n = user_callback_handle_rxflow(wsi->a.protocol->callback,
 			wsi, LWS_CALLBACK_RAW_WRITEABLE,
 			wsi->user_space, NULL, 0);

lib/roles/ws/client-ws.c

@@ -251,11 +251,6 @@ lws_client_ws_upgrade(struct lws *wsi, const char **cce)
 	char ignore;
 #endif
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-		wsi->detlat.earliest_write_req = 0;
-		wsi->detlat.earliest_write_req_pre_write = 0;
-#endif
-
 	if (wsi->client_mux_substream) {/* !!! client ws-over-h2 not there yet */
 		lwsl_warn("%s: client ws-over-h2 upgrade not supported yet\n",
 			  __func__);

lib/secure-streams/policy-common.c

@@ -297,6 +297,17 @@ lws_ss_policy_set(struct lws_context *context, const char *name)
 	if (context->ac_policy) {
 		int n;
 
+#if defined(LWS_WITH_SYS_METRICS)
+		lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1,
+					   context->owner_mtr_dynpol.head) {
+			lws_metric_policy_dyn_t *dm =
+				lws_container_of(d, lws_metric_policy_dyn_t, list);
+
+			lws_metric_policy_dyn_destroy(dm, 1); /* keep */
+
+		} lws_end_foreach_dll_safe(d, d1);
+#endif
+
 		/*
 		 * any existing ss created with the old policy have to go away
 		 * now, since they point to the shortly-to-be-destroyed old
@@ -431,10 +442,27 @@ lws_ss_policy_set(struct lws_context *context, const char *name)
 		x = x->next;
 	}
 
+	context->last_policy = time(NULL);
+#if defined(LWS_WITH_SYS_METRICS)
+	if (context->pss_policies)
+		((lws_ss_policy_t *)context->pss_policies)->metrics =
+						args->heads[LTY_METRICS].m;
+#endif
+
 	/* and we can discard the parsing args object now, invalidating args */
 
 	lws_free_set_NULL(context->pol_args);
 #endif
 
+#if defined(LWS_WITH_SYS_METRICS)
+	lws_metric_rebind_policies(context);
+#endif
+
+#if defined(LWS_WITH_SYS_SMD)
+	(void)lws_smd_msg_printf(context, LWSSMDCL_SYSTEM_STATE,
+				 "{\"policy\":\"updated\",\"ts\":%lu}",
+				   (long)context->last_policy);
+#endif
+
 	return ret;
 }

lib/secure-streams/policy-json.c

@@ -41,6 +41,11 @@ static const char * const lejp_tokens_policy[] = {
 	"certs[].*",
 	"trust_stores[].name",
 	"trust_stores[].stack",
+	"metrics[].name",
+	"metrics[].us_schedule",
+	"metrics[].us_halflife",
+	"metrics[].min_outlier",
+	"metrics[].report",
 	"s[].*.endpoint",
 	"s[].*.via-socks5",
 	"s[].*.protocol",
@@ -135,6 +140,11 @@ typedef enum {
 	LSSPPT_CERTS,
 	LSSPPT_TRUST_STORES_NAME,
 	LSSPPT_TRUST_STORES_STACK,
+	LSSPPT_METRICS_NAME,
+	LSSPPT_METRICS_US_SCHEDULE,
+	LSSPPT_METRICS_US_HALFLIFE,
+	LSSPPT_METRICS_MIN_OUTLIER,
+	LSSPPT_METRICS_REPORT,
 	LSSPPT_ENDPOINT,
 	LSSPPT_VH_VIA_SOCKS5,
 	LSSPPT_PROTOCOL,
@@ -225,6 +235,7 @@ static uint16_t sizes[] = {
 	sizeof(lws_ss_trust_store_t),
 	sizeof(lws_ss_policy_t),
 	sizeof(lws_ss_auth_t),
+	sizeof(lws_metric_policy_t),
 };
 
 static const char * const protonames[] = {
@@ -253,6 +264,25 @@ lws_ss_policy_find_auth_by_name(struct policy_cb_args *a,
 	return NULL;
 }
 
+static int
+lws_ss_policy_alloc_helper(struct policy_cb_args *a, int type)
+{
+	/*
+	 * We do the pointers always as .b union member, all of the
+	 * participating structs begin with .next and .name the same
+	 */
+
+	a->curr[type].b = lwsac_use_zero(&a->ac,
+				sizes[type], POL_AC_GRAIN);
+	if (!a->curr[type].b)
+		return 1;
+
+	a->curr[type].b->next = a->heads[type].b;
+	a->heads[type].b = a->curr[type].b;
+
+	return 0;
+}
+
 static signed char
 lws_ss_policy_parser_cb(struct lejp_ctx *ctx, char reason)
 {
@@ -291,6 +321,13 @@ lws_ss_policy_parser_cb(struct lejp_ctx *ctx, char reason)
 	case LSSPPT_AUTH:
 		n = LTY_AUTH;
 		break;
+	case LSSPPT_METRICS_NAME:
+	case LSSPPT_METRICS_US_SCHEDULE:
+	case LSSPPT_METRICS_US_HALFLIFE:
+	case LSSPPT_METRICS_MIN_OUTLIER:
+	case LSSPPT_METRICS_REPORT:
+		n = LTY_METRICS;
+		break;
 	}
 
 	if (reason == LEJPCB_ARRAY_START &&
@@ -300,13 +337,8 @@ lws_ss_policy_parser_cb(struct lejp_ctx *ctx, char reason)
 		a->count = 0;
 
 	if (reason == LEJPCB_OBJECT_START && n == LTY_AUTH) {
-		a->curr[n].b = lwsac_use_zero(&a->ac, sizes[n], POL_AC_GRAIN);
+		if (lws_ss_policy_alloc_helper(a, LTY_AUTH))
-		if (!a->curr[n].b)
 			goto oom;
-
-		a->curr[n].b->next = a->heads[n].b;
-		a->heads[n].b = a->curr[n].b;
-
 		return 0;
 	}
 
@@ -360,7 +392,7 @@ lws_ss_policy_parser_cb(struct lejp_ctx *ctx, char reason)
 	}
 
 	if (reason == LEJPCB_PAIR_NAME && n != -1 &&
-	    (n != LTY_TRUSTSTORE && n != LTY_AUTH)) {
+	    (n != LTY_TRUSTSTORE && n != LTY_AUTH && n != LTY_METRICS)) {
 
 		p2 = NULL;
 		if (n == LTY_POLICY) {
@@ -490,18 +522,10 @@ lws_ss_policy_parser_cb(struct lejp_ctx *ctx, char reason)
 		break;
 
 	case LSSPPT_TRUST_STORES_NAME:
-		/*
+		if (lws_ss_policy_alloc_helper(a, LTY_TRUSTSTORE))
-		 * We do the pointers always as .b, all of the participating
-		 * structs begin with .next and .name
-		 */
-		a->curr[LTY_TRUSTSTORE].b = lwsac_use_zero(&a->ac,
-					sizes[LTY_TRUSTSTORE], POL_AC_GRAIN);
-		if (!a->curr[LTY_TRUSTSTORE].b)
 			goto oom;
 
 		a->count = 0;
-		a->curr[LTY_TRUSTSTORE].b->next = a->heads[LTY_TRUSTSTORE].b;
-		a->heads[LTY_TRUSTSTORE].b = a->curr[LTY_TRUSTSTORE].b;
 		pp = (char **)&a->curr[LTY_TRUSTSTORE].b->name;
 
 		goto string2;
@@ -528,6 +552,31 @@ lws_ss_policy_parser_cb(struct lejp_ctx *ctx, char reason)
 		lwsl_err("%s: unknown trust store entry %s\n", __func__,
 			 dotstar);
 		goto oom;
+#if defined(LWS_WITH_SYS_METRICS)
+	case LSSPPT_METRICS_NAME:
+		if (lws_ss_policy_alloc_helper(a, LTY_METRICS))
+			goto oom;
+
+		pp = (char **)&a->curr[LTY_METRICS].b->name;
+
+		goto string2;
+
+	case LSSPPT_METRICS_US_SCHEDULE:
+		a->curr[LTY_METRICS].m->us_schedule = (uint32_t)atol(ctx->buf);
+		break;
+
+	case LSSPPT_METRICS_US_HALFLIFE:
+		a->curr[LTY_METRICS].m->us_decay_unit = (uint32_t)atol(ctx->buf);
+		break;
+
+	case LSSPPT_METRICS_MIN_OUTLIER:
+		a->curr[LTY_METRICS].m->min_contributors = (uint8_t)atoi(ctx->buf);
+		break;
+
+	case LSSPPT_METRICS_REPORT:
+		pp = (char **)&a->curr[LTY_METRICS].m->report;
+		goto string2;
+#endif
 
 	case LSSPPT_SERVER_CERT:
 	case LSSPPT_SERVER_KEY:

lib/secure-streams/private-lib-secure-streams.h

@@ -49,6 +49,10 @@ typedef struct lws_ss_handle {
 
 	lws_lifecycle_t		lc;
 
+#if defined(LWS_WITH_SYS_METRICS)
+	lws_metrics_caliper_compose(cal_txn)
+#endif
+
 	struct lws_dll2		list;	  /**< pt lists active ss */
 	struct lws_dll2		to_list;  /**< pt lists ss with pending to-s */
 #if defined(LWS_WITH_SERVER)
@@ -290,6 +294,10 @@ typedef struct lws_sspc_handle {
 	struct lws_dll2		client_list;
 	struct lws_tx_credit	txc;
 
+#if defined(LWS_WITH_SYS_METRICS)
+	lws_metrics_caliper_compose(cal_txn)
+#endif
+
 	struct lws		*cwsi;
 
 	struct lws_dsh		*dsh;
@@ -333,6 +341,7 @@ union u {
 	lws_ss_trust_store_t	*t;
 	lws_ss_policy_t		*p;
 	lws_ss_auth_t		*a;
+	lws_metric_policy_t	*m;
 };
 
 enum {
@@ -341,6 +350,7 @@ enum {
 	LTY_TRUSTSTORE,
 	LTY_POLICY,
 	LTY_AUTH,
+	LTY_METRICS,
 
 	_LTY_COUNT /* always last */
 };
@@ -510,6 +520,29 @@ struct ss_pcols {
 	secstream_protocol_get_txcr_t			tx_cr_est;
 };
 
+/*
+ * Because both sides of the connection share the conn, we allocate it
+ * during accepted adoption, and both sides point to it.
+ *
+ * When .ss or .wsi close, they must NULL their entry here so no dangling
+ * refereneces.
+ *
+ * The last one of the accepted side and the onward side to close frees it.
+ */
+
+
+
+
+struct conn {
+	struct lws_ss_serialization_parser parser;
+
+	lws_dsh_t		*dsh;	/* unified buffer for both sides */
+	struct lws		*wsi;	/* the proxy's client side */
+	lws_ss_handle_t		*ss;	/* the onward, ss side */
+
+	lws_ss_conn_states_t	state;
+};
+
 extern const struct ss_pcols ss_pcol_h1;
 extern const struct ss_pcols ss_pcol_h2;
 extern const struct ss_pcols ss_pcol_ws;

lib/secure-streams/protocols/ss-h1.c

@@ -415,6 +415,7 @@ secstream_h1(struct lws *wsi, enum lws_callback_reasons reason, void *user,
 			break;
 		}
 		assert(h->policy);
+		lws_metrics_caliper_report_hist(h->cal_txn, wsi);
 		lwsl_info("%s: %s CLIENT_CONNECTION_ERROR: %s\n", __func__,
 			  h->lc.gutag, in ? (const char *)in : "none");
 		/* already disconnected, no action for DISCONNECT_ME */
@@ -445,8 +446,11 @@ secstream_h1(struct lws *wsi, enum lws_callback_reasons reason, void *user,
 			break;
 
 		lws_sul_cancel(&h->sul_timeout);
-		lwsl_notice("%s: %s LWS_CALLBACK_CLOSED_CLIENT_HTTP\n",
+
-				__func__, wsi->lc.gutag);
+		lws_metrics_caliper_report_hist(h->cal_txn, wsi);
+		//lwsl_notice("%s: %s LWS_CALLBACK_CLOSED_CLIENT_HTTP\n",
+		//		__func__, wsi->lc.gutag);
+
 		h->wsi = NULL;
 
 #if defined(LWS_WITH_SERVER)
@@ -487,6 +491,13 @@ secstream_h1(struct lws *wsi, enum lws_callback_reasons reason, void *user,
 			/* it's just telling use we connected / joined the nwsi */
 	//		break;
 
+#if defined(LWS_WITH_SYS_METRICS)
+		if (status) {
+			lws_snprintf((char *)buf, 10, "%d", status);
+			lws_metrics_tag_ss_add(h, "http_resp", (char *)buf);
+		}
+#endif
+
 		if (status == HTTP_STATUS_SERVICE_UNAVAILABLE /* 503 */ ||
 		    status == 429 /* Too many requests */) {
 			/*
@@ -552,6 +563,7 @@ secstream_h1(struct lws *wsi, enum lws_callback_reasons reason, void *user,
 		lws_sul_cancel(&h->sul);
 
 		if (h->prev_ss_state != LWSSSCS_CONNECTED) {
+			lws_metrics_caliper_report_hist(h->cal_txn, wsi);
 			r = lws_ss_event_helper(h, LWSSSCS_CONNECTED);
 			if (r != LWSSSSRET_OK)
 				return _lws_ss_handle_state_ret_CAN_DESTROY_HANDLE(r, wsi, &h);
@@ -722,6 +734,12 @@ malformed:
 		     h->being_serialized && (
 				!strcmp(h->policy->u.http.method, "PUT") ||
 				!strcmp(h->policy->u.http.method, "POST"))) {
+#if defined(LWS_WITH_SYS_METRICS)
+			/*
+			 * If any hanging caliper measurement, dump it, and free any tags
+			 */
+			lws_metrics_caliper_report_hist(h->cal_txn, (struct lws *)NULL);
+#endif
 			r = lws_ss_event_helper(h, LWSSSCS_CONNECTED);
 			if (r)
 				return _lws_ss_handle_state_ret_CAN_DESTROY_HANDLE(r, wsi, &h);
@@ -773,7 +791,7 @@ malformed:
 		return 0; /* don't passthru */
 
 	case LWS_CALLBACK_COMPLETED_CLIENT_HTTP:
-		lwsl_debug("%s: LWS_CALLBACK_COMPLETED_CLIENT_HTTP\n", __func__);
+		// lwsl_debug("%s: LWS_CALLBACK_COMPLETED_CLIENT_HTTP\n", __func__);
 
 		if (!h)
 			return -1;
@@ -919,7 +937,7 @@ malformed:
 		}
 
 #if defined(LWS_WITH_SERVER)
-		if (!(h->info.flags & LWSSSINFLAGS_ACCEPTED) &&
+		if ((h->info.flags & LWSSSINFLAGS_ACCEPTED) /* server */ &&
 		    (f & LWSSS_FLAG_EOM) &&
 		     lws_http_transaction_completed(wsi))
 			return -1;
@@ -973,6 +991,12 @@ malformed:
 		}
 
 		if (!h->ss_dangling_connected) {
+#if defined(LWS_WITH_SYS_METRICS)
+			/*
+			 * If any hanging caliper measurement, dump it, and free any tags
+			 */
+			lws_metrics_caliper_report_hist(h->cal_txn, (struct lws *)NULL);
+#endif
 			r = lws_ss_event_helper(h, LWSSSCS_CONNECTED);
 			if (r)
 				return _lws_ss_handle_state_ret_CAN_DESTROY_HANDLE(r, wsi, &h);

lib/secure-streams/protocols/ss-mqtt.c

@@ -94,6 +94,12 @@ secstream_mqtt(struct lws *wsi, enum lws_callback_reasons reason, void *user,
 		h->retry = 0;
 		h->seqstate = SSSEQ_CONNECTED;
 		lws_sul_cancel(&h->sul);
+#if defined(LWS_WITH_SYS_METRICS)
+		/*
+		 * If any hanging caliper measurement, dump it, and free any tags
+		 */
+		lws_metrics_caliper_report_hist(h->cal_txn, (struct lws *)NULL);
+#endif
 		r = lws_ss_event_helper(h, LWSSSCS_CONNECTED);
 		if (r != LWSSSSRET_OK)
 			return _lws_ss_handle_state_ret_CAN_DESTROY_HANDLE(r, wsi, &h);

lib/secure-streams/protocols/ss-raw.c

@@ -60,8 +60,7 @@ secstream_raw(struct lws *wsi, enum lws_callback_reasons reason, void *user,
 		if (!h)
 			break;
 		lws_sul_cancel(&h->sul_timeout);
-		lwsl_info("%s: %s, %s LWS_CALLBACK_CLOSED_CLIENT_HTTP\n",
+		lwsl_info("%s: %s, %s RAW_CLOSE\n", __func__, lws_ss_tag(h),
-			  __func__, lws_ss_tag(h),
 			  h->policy ? h->policy->streamtype : "no policy");
 		h->wsi = NULL;
 #if defined(LWS_WITH_SERVER)
@@ -69,6 +68,12 @@ secstream_raw(struct lws *wsi, enum lws_callback_reasons reason, void *user,
 		lws_dll2_remove(&h->cli_list);
 		lws_pt_unlock(pt);
 #endif
+
+		/* wsi is going down anyway */
+		r = lws_ss_event_helper(h, LWSSSCS_DISCONNECTED);
+		if (r == LWSSSSRET_DESTROY_ME)
+			return _lws_ss_handle_state_ret_CAN_DESTROY_HANDLE(r, wsi, &h);
+
 		if (h->policy && !(h->policy->flags & LWSSSPOLF_OPPORTUNISTIC) &&
 #if defined(LWS_WITH_SERVER)
 			    !(h->info.flags & LWSSSINFLAGS_ACCEPTED) && /* not server */
@@ -79,10 +84,7 @@ secstream_raw(struct lws *wsi, enum lws_callback_reasons reason, void *user,
 				return _lws_ss_handle_state_ret_CAN_DESTROY_HANDLE(r, wsi, &h);
 			break;
 		}
-		/* wsi is going down anyway */
+
-		r = lws_ss_event_helper(h, LWSSSCS_DISCONNECTED);
-		if (r == LWSSSSRET_DESTROY_ME)
-			return _lws_ss_handle_state_ret_CAN_DESTROY_HANDLE(r, wsi, &h);
 		break;
 
 	case LWS_CALLBACK_RAW_CONNECTED:
@@ -91,6 +93,12 @@ secstream_raw(struct lws *wsi, enum lws_callback_reasons reason, void *user,
 		h->retry = 0;
 		h->seqstate = SSSEQ_CONNECTED;
 		lws_sul_cancel(&h->sul);
+#if defined(LWS_WITH_SYS_METRICS)
+		/*
+		 * If any hanging caliper measurement, dump it, and free any tags
+		 */
+		lws_metrics_caliper_report_hist(h->cal_txn, (struct lws *)NULL);
+#endif
 		r = lws_ss_event_helper(h, LWSSSCS_CONNECTED);
 		if (r != LWSSSSRET_OK)
 			return _lws_ss_handle_state_ret_CAN_DESTROY_HANDLE(r, wsi, &h);

lib/secure-streams/protocols/ss-ws.c

@@ -106,6 +106,12 @@ secstream_ws(struct lws *wsi, enum lws_callback_reasons reason, void *user,
 		h->retry = 0;
 		h->seqstate = SSSEQ_CONNECTED;
 		lws_sul_cancel(&h->sul);
+#if defined(LWS_WITH_SYS_METRICS)
+		/*
+		 * If any hanging caliper measurement, dump it, and free any tags
+		 */
+		lws_metrics_caliper_report_hist(h->cal_txn, (struct lws *)NULL);
+#endif
 		r = lws_ss_event_helper(h, LWSSSCS_CONNECTED);
 		if (r != LWSSSSRET_OK)
 			return _lws_ss_handle_state_ret_CAN_DESTROY_HANDLE(r, wsi, &h);

lib/secure-streams/secure-streams-client.c

@@ -71,9 +71,23 @@ lws_sspc_sul_retry_cb(lws_sorted_usec_list_t *sul)
 	i.pwsi = &h->cwsi;
 	i.opaque_user_data = (void *)h;
 	i.ssl_connection = LCCSCF_SECSTREAM_PROXY_LINK;
+
+	lws_metrics_caliper_bind(h->cal_txn, h->context->mt_ss_cliprox_conn);
+#if defined(LWS_WITH_SYS_METRICS)
+	lws_metrics_tag_add(&h->cal_txn.mtags_owner, "ss", h->ssi.streamtype);
+#endif
+
 	/* this wsi is the link to the proxy */
 
 	if (!lws_client_connect_via_info(&i)) {
+
+#if defined(LWS_WITH_SYS_METRICS)
+		/*
+		 * If any hanging caliper measurement, dump it, and free any tags
+		 */
+		lws_metrics_caliper_report_hist(h->cal_txn, (struct lws *)NULL);
+#endif
+
 		lws_sul_schedule(h->context, 0, &h->sul_retry,
 				 lws_sspc_sul_retry_cb, LWS_US_PER_SEC);
 
@@ -147,6 +161,12 @@ callback_sspc_client(struct lws *wsi, enum lws_callback_reasons reason,
 
 	case LWS_CALLBACK_CLIENT_CONNECTION_ERROR:
 		lwsl_warn("%s: CONNECTION_ERROR\n", __func__);
+#if defined(LWS_WITH_SYS_METRICS)
+		/*
+		 * If any hanging caliper measurement, dump it, and free any tags
+		 */
+		lws_metrics_caliper_report_hist(h->cal_txn, (struct lws *)NULL);
+#endif
 		lws_set_opaque_user_data(wsi, NULL);
 		h->cwsi = NULL;
 		lws_sul_schedule(h->context, 0, &h->sul_retry,
@@ -600,6 +620,13 @@ lws_sspc_destroy(lws_sspc_handle_t **ph)
 
 	h->destroying = 1;
 
+	/* if this caliper is still dangling at destroy, we failed */
+#if defined(LWS_WITH_SYS_METRICS)
+	/*
+	 * If any hanging caliper measurement, dump it, and free any tags
+	 */
+	lws_metrics_caliper_report_hist(h->cal_txn, (struct lws *)NULL);
+#endif
 	if (h->ss_dangling_connected && h->ssi.state) {
 		lws_sspc_event_helper(h, LWSSSCS_DISCONNECTED, 0);
 		h->ss_dangling_connected = 0;

lib/secure-streams/secure-streams-process.c

@@ -51,26 +51,6 @@
 
 #include <private-lib-core.h>
 
-/*
- * Because both sides of the connection share the conn, we allocate it
- * during accepted adoption, and both sides point to it.
- *
- * When .ss or .wsi close, they must NULL their entry here so no dangling
- * refereneces.
- *
- * The last one of the accepted side and the onward side to close frees it.
- */
-
-struct conn {
-	struct lws_ss_serialization_parser parser;
-
-	lws_dsh_t		*dsh;	/* unified buffer for both sides */
-	struct lws		*wsi;	/* the proxy's client side */
-	lws_ss_handle_t		*ss;	/* the onward, ss side */
-
-	lws_ss_conn_states_t	state;
-};
-
 struct raw_pss {
 	struct conn		*conn;
 };
@@ -313,9 +293,6 @@ callback_ss_proxy(struct lws *wsi, enum lws_callback_reasons reason,
 	lws_ss_metadata_t *md;
 	lws_ss_info_t ssi;
 	const uint8_t *cp;
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	lws_usec_t us;
-#endif
 	char s[256];
 	uint8_t *p;
 	size_t si;
@@ -588,7 +565,7 @@ callback_ss_proxy(struct lws *wsi, enum lws_callback_reasons reason,
 
 			cp = p;
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
+#if 0
 			if (cp[0] == LWSSS_SER_RXPRE_RX_PAYLOAD &&
 			    wsi->a.context->detailed_latency_cb) {
 

lib/secure-streams/secure-streams-serialize.c

@@ -224,7 +224,7 @@ lws_ss_deserialize_tx_payload(struct lws_dsh *dsh, struct lws *wsi,
 
 	*flags = (int)lws_ser_ru32be(&p[3]);
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
+#if 0
 	if (wsi && wsi->a.context->detailed_latency_cb) {
 		/*
 		 * use the proxied latency information to compute the client
@@ -725,7 +725,7 @@ payload_ff:
 					}
 				}
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
+#if 0
 				if (lws_det_lat_active(context)) {
 					lws_detlat_t d;
 
@@ -1233,6 +1233,13 @@ payload_ff:
 			 * CREATING now so we'll know the metadata to sync.
 			 */
 
+#if defined(LWS_WITH_SYS_METRICS)
+			/*
+			 * If any hanging caliper measurement, dump it, and free any tags
+			 */
+			lws_metrics_caliper_report_hist(h->cal_txn, (struct lws *)NULL);
+#endif
+
 			if (!h->creating_cb_done) {
 				if (lws_ss_check_next_state(&h->lc, &h->prev_ss_state,
 							    LWSSSCS_CREATING))

lib/secure-streams/secure-streams.c

@@ -159,6 +159,7 @@ static const uint32_t ss_state_txn_validity[] = {
 					  (1 << LWSSSCS_POLL) |
 					  (1 << LWSSSCS_TIMEOUT) |
 					  (1 << LWSSSCS_DISCONNECTED) |
+					  (1 << LWSSSCS_UNREACHABLE) |
 					  (1 << LWSSSCS_DESTROYING),
 
 	[LWSSSCS_SERVER_TXN]		= (1 << LWSSSCS_DISCONNECTED) |
@@ -671,6 +672,14 @@ _lws_ss_client_connect(lws_ss_handle_t *h, int is_retry, void *conn_if_sspc_onw)
 	lwsl_info("%s: connecting %s, '%s' '%s' %s\n", __func__, i.method,
 			i.alpn, i.address, i.path);
 
+#if defined(LWS_WITH_SYS_METRICS)
+	/* possibly already hanging connect retry... */
+	if (!h->cal_txn.mt)
+		lws_metrics_caliper_bind(h->cal_txn, h->context->mth_ss_conn);
+
+	lws_metrics_tag_add(&h->cal_txn.mtags_owner, "ss", h->policy->streamtype);
+#endif
+
 	h->txn_ok = 0;
 	r = lws_ss_event_helper(h, LWSSSCS_CONNECTING);
 	if (r) {
@@ -1181,6 +1190,13 @@ lws_ss_destroy(lws_ss_handle_t **ppss)
 	lws_fi_destroy(&h->fi);
 #endif
 
+#if defined(LWS_WITH_SYS_METRICS)
+	/*
+	 * If any hanging caliper measurement, dump it, and free any tags
+	 */
+	lws_metrics_caliper_report_hist(h->cal_txn, (struct lws *)NULL);
+#endif
+
 	lws_sul_cancel(&h->sul_timeout);
 
 	/* confirm no sul left scheduled in handle or user allocation object */

lib/system/CMakeLists.txt

@@ -63,6 +63,8 @@ if (LWS_WITH_NETWORK)
 			system/fault-injection/fault-injection.c)
 	endif()
 
+	add_subdir_include_dirs(metrics)
+
 endif()
 
 #

lib/system/async-dns/async-dns-parse.c

@@ -1,7 +1,7 @@
 /*
  * libwebsockets - small server side websockets and web server implementation
  *
- * Copyright (C) 2010 - 2019 Andy Green <andy@warmcat.com>
+ * Copyright (C) 2010 - 2021 Andy Green <andy@warmcat.com>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to
@@ -690,6 +690,8 @@ lws_adns_parse_udp(lws_async_dns_t *dns, const uint8_t *pkt, size_t len)
 	c->incomplete = 0;
 	lws_async_dns_complete(q, q->firstcache);
 
+	q->go_nogo = METRES_GO;
+
 	/*
 	 * the query is completely finished with
 	 */

lib/system/async-dns/async-dns.c

@@ -34,6 +34,8 @@ static const lws_retry_bo_t retry_policy = {
 void
 lws_adns_q_destroy(lws_adns_q_t *q)
 {
+	lws_metrics_caliper_report(q->metcal, (char)q->go_nogo);
+
 	lws_sul_cancel(&q->sul);
 	lws_sul_cancel(&q->write_sul);
 	lws_dll2_remove(&q->list);
@@ -703,6 +705,10 @@ lws_async_dns_query(struct lws_context *context, int tsi, const char *name,
 		if (c->results)
 			c->refcount++;
 
+#if defined(LWS_WITH_SYS_METRICS)
+		lws_metric_event(context->mt_adns_cache,  METRES_GO, 0);
+#endif
+
 		if (cb(wsi, name, c->results, m, opaque) == NULL)
 			return LADNS_RET_FAILED_WSI_CLOSED;
 
@@ -710,6 +716,10 @@ lws_async_dns_query(struct lws_context *context, int tsi, const char *name,
 	} else
 		lwsl_info("%s: %s uncached\n", __func__, name);
 
+#if defined(LWS_WITH_SYS_METRICS)
+	lws_metric_event(context->mt_adns_cache, METRES_NOGO, 0);
+#endif
+
 	/*
 	 * It's a 1.2.3.4 or ::1 type IP address already?  We don't need a dns
 	 * server set up to be able to create an addrinfo result for that.
@@ -876,6 +886,10 @@ lws_async_dns_query(struct lws_context *context, int tsi, const char *name,
 
 	lws_dll2_add_head(&q->list, &dns->waiting);
 
+	lws_metrics_caliper_bind(q->metcal, context->mt_conn_dns);
+	q->go_nogo = METRES_NOGO;
+	/* caliper is reported in lws_adns_q_destroy */
+
 	lwsl_info("%s: created new query: %s\n", __func__, name);
 	lws_adns_dump(dns);
 

lib/system/async-dns/private-lib-async-dns.h

@@ -58,6 +58,8 @@ typedef struct {
 	lws_sorted_usec_list_t	write_sul;	/* fail if unable to write by this time */
 	lws_dll2_t		list;
 
+	lws_metrics_caliper_compose(metcal)
+
 	lws_dll2_owner_t	wsi_adns;
 	lws_async_dns_cb_t	standalone_cb;	/* if not associated to wsi */
 	struct lws_context	*context;
@@ -83,6 +85,7 @@ typedef struct {
 
 	uint8_t			recursion;
 	uint8_t			tids;
+	uint8_t			go_nogo;
 
 	uint8_t			is_retry:1;
 

lib/system/metrics/CMakeLists.txt

@@ -0,0 +1,10 @@
+include_directories(.)
+
+if (LWS_WITH_SYS_METRICS)
+	list(APPEND SOURCES
+		system/metrics/metrics.c
+	)
+endif()
+
+exports_to_parent_scope()
+

lib/system/metrics/metrics.c

@@ -0,0 +1,891 @@
+/*
+ * lws Generic Metrics
+ *
+ * Copyright (C) 2019 - 2021 Andy Green <andy@warmcat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "private-lib-core.h"
+#include <assert.h>
+
+int
+lws_metrics_tag_add(lws_dll2_owner_t *owner, const char *name, const char *val)
+{
+	size_t vl = strlen(val);
+	lws_metrics_tag_t *tag;
+
+	// lwsl_notice("%s: adding %s=%s\n", __func__, name, val);
+
+	/*
+	 * Remove (in order to replace) any existing tag of same name
+	 */
+
+	lws_start_foreach_dll(struct lws_dll2 *, d, owner->head) {
+		tag = lws_container_of(d, lws_metrics_tag_t, list);
+
+		if (!strcmp(name, tag->name)) {
+			lws_dll2_remove(&tag->list);
+			lws_free(tag);
+			break;
+		}
+
+	} lws_end_foreach_dll(d);
+
+	/*
+	 * Create the new tag
+	 */
+
+	tag = lws_malloc(sizeof(*tag) + vl + 1, __func__);
+	if (!tag)
+		return 1;
+
+	lws_dll2_clear(&tag->list);
+	tag->name = name;
+	memcpy(&tag[1], val, vl + 1);
+
+	lws_dll2_add_tail(&tag->list, owner);
+
+	return 0;
+}
+
+int
+lws_metrics_tag_wsi_add(struct lws *wsi, const char *name, const char *val)
+{
+	__lws_lc_tag(NULL, &wsi->lc, "|%s", val);
+
+	return lws_metrics_tag_add(&wsi->cal_conn.mtags_owner, name, val);
+}
+
+#if defined(LWS_WITH_SECURE_STREAMS)
+int
+lws_metrics_tag_ss_add(struct lws_ss_handle *ss, const char *name, const char *val)
+{
+	__lws_lc_tag(NULL, &ss->lc, "|%s", val);
+	return lws_metrics_tag_add(&ss->cal_txn.mtags_owner, name, val);
+}
+#if defined(LWS_WITH_SECURE_STREAMS)
+int
+lws_metrics_tag_sspc_add(struct lws_sspc_handle *sspc, const char *name,
+			 const char *val)
+{
+	__lws_lc_tag(NULL, &sspc->lc, "|%s", val);
+	return lws_metrics_tag_add(&sspc->cal_txn.mtags_owner, name, val);
+}
+#endif
+#endif
+
+void
+lws_metrics_tags_destroy(lws_dll2_owner_t *owner)
+{
+	lws_metrics_tag_t *t;
+
+	lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1, owner->head) {
+		t = lws_container_of(d, lws_metrics_tag_t, list);
+
+		lws_dll2_remove(&t->list);
+		lws_free(t);
+
+	} lws_end_foreach_dll_safe(d, d1);
+}
+
+size_t
+lws_metrics_tags_serialize(lws_dll2_owner_t *owner, char *buf, size_t len)
+{
+	char *end = buf + len - 1, *p = buf;
+	lws_metrics_tag_t *t;
+
+	lws_start_foreach_dll(struct lws_dll2 *, d, owner->head) {
+		t = lws_container_of(d, lws_metrics_tag_t, list);
+
+		p += lws_snprintf(p, lws_ptr_diff_size_t(end, p),
+				  "%s=\"%s\"", t->name, (const char *)&t[1]);
+
+		if (d->next && p + 2 < end)
+			*p++ = ',';
+
+	} lws_end_foreach_dll(d);
+
+	*p = '\0';
+
+	return lws_ptr_diff_size_t(p, buf);
+}
+
+const char *
+lws_metrics_tag_get(lws_dll2_owner_t *owner, const char *name)
+{
+	lws_metrics_tag_t *t;
+
+	lws_start_foreach_dll(struct lws_dll2 *, d, owner->head) {
+		t = lws_container_of(d, lws_metrics_tag_t, list);
+
+		if (!strcmp(name, t->name))
+			return (const char *)&t[1];
+
+	} lws_end_foreach_dll(d);
+
+	return NULL;
+}
+
+static int
+lws_metrics_dump_cb(lws_metric_pub_t *pub, void *user);
+
+static void
+lws_metrics_report_and_maybe_clear(struct lws_context *ctx, lws_metric_pub_t *pub)
+{
+	if (!pub->us_first || pub->us_last == pub->us_dumped)
+		return;
+
+	lws_metrics_dump_cb(pub, ctx);
+}
+
+static void
+lws_metrics_periodic_cb(lws_sorted_usec_list_t *sul)
+{
+	lws_metric_policy_dyn_t *dmp = lws_container_of(sul,
+						lws_metric_policy_dyn_t, sul);
+	struct lws_context *ctx = lws_container_of(dmp->list.owner,
+					struct lws_context, owner_mtr_dynpol);
+
+	if (!ctx->system_ops || !ctx->system_ops->metric_report)
+		return;
+
+	lws_start_foreach_dll(struct lws_dll2 *, d, dmp->owner.head) {
+		lws_metric_t *mt = lws_container_of(d, lws_metric_t, list);
+		lws_metric_pub_t *pub = lws_metrics_priv_to_pub(mt);
+
+		lws_metrics_report_and_maybe_clear(ctx, pub);
+
+	} lws_end_foreach_dll(d);
+
+#if defined(LWS_WITH_SYS_SMD) && defined(LWS_WITH_SECURE_STREAMS)
+	(void)lws_smd_msg_printf(ctx, LWSSMDCL_METRICS,
+				 "{\"dump\":\"%s\",\"ts\":%lu}",
+				   dmp->policy->name,
+				   (long)ctx->last_policy);
+#endif
+
+	if (dmp->policy->us_schedule)
+		lws_sul_schedule(ctx, 0, &dmp->sul,
+				 lws_metrics_periodic_cb,
+				 dmp->policy->us_schedule);
+}
+
+/*
+ * Policies are in two pieces, a const policy and a dynamic part that contains
+ * lists and sul timers for the policy etc.  This creates a dynmic part
+ * corresponding to the static part.
+ *
+ * Metrics can exist detached from being bound to any policy about how to
+ * report them, these are collected but not reported unless they later become
+ * bound to a reporting policy dynamically.
+ */
+
+lws_metric_policy_dyn_t *
+lws_metrics_policy_dyn_create(struct lws_context *ctx,
+			      const lws_metric_policy_t *po)
+{
+	lws_metric_policy_dyn_t *dmet;
+
+	dmet = lws_zalloc(sizeof(*dmet), __func__);
+	if (!dmet)
+		return NULL;
+
+	dmet->policy = po;
+	lws_dll2_add_tail(&dmet->list, &ctx->owner_mtr_dynpol);
+
+	if (po->us_schedule)
+		lws_sul_schedule(ctx, 0, &dmet->sul,
+				 lws_metrics_periodic_cb,
+				 po->us_schedule);
+
+	return dmet;
+}
+
+/*
+ * Get a dynamic metrics policy from the const one, may return NULL if OOM
+ */
+
+lws_metric_policy_dyn_t *
+lws_metrics_policy_get_dyn(struct lws_context *ctx,
+			   const lws_metric_policy_t *po)
+{
+	lws_start_foreach_dll(struct lws_dll2 *, d, ctx->owner_mtr_dynpol.head) {
+		lws_metric_policy_dyn_t *dm =
+			lws_container_of(d, lws_metric_policy_dyn_t, list);
+
+		if (dm->policy == po)
+			return dm;
+
+	} lws_end_foreach_dll(d);
+
+	/*
+	 * no dyn policy part for this const policy --> create one
+	 *
+	 * We want a dynamic part for listing metrics that bound to the policy
+	 */
+
+	return lws_metrics_policy_dyn_create(ctx, po);
+}
+
+static int
+lws_metrics_check_in_policy(const char *polstring, const char *name)
+{
+	struct lws_tokenize ts;
+
+	memset(&ts, 0, sizeof(ts));
+
+	ts.start = polstring;
+	ts.len = strlen(polstring);
+	ts.flags = (uint16_t)(LWS_TOKENIZE_F_MINUS_NONTERM |
+			      LWS_TOKENIZE_F_ASTERISK_NONTERM |
+			      LWS_TOKENIZE_F_COMMA_SEP_LIST |
+			      LWS_TOKENIZE_F_NO_FLOATS |
+			      LWS_TOKENIZE_F_DOT_NONTERM);
+
+	do {
+		ts.e = (int8_t)lws_tokenize(&ts);
+
+		if (ts.e == LWS_TOKZE_TOKEN) {
+			if (!lws_strcmp_wildcard(ts.token, ts.token_len, name))
+				/* yes, we are mentioned in this guy's policy */
+				return 0;
+		}
+	} while (ts.e > 0);
+
+	/* no, this policy doesn't apply to a metric with our name */
+
+	return 1;
+}
+
+static const lws_metric_policy_t *
+lws_metrics_find_policy(struct lws_context *ctx, const char *name)
+{
+	const lws_metric_policy_t *mp = ctx->metrics_policies;
+
+	if (!mp) {
+#if defined(LWS_WITH_SECURE_STREAMS)
+		if (ctx->pss_policies)
+			mp = ctx->pss_policies->metrics;
+#endif
+		if (!mp)
+			return NULL;
+	}
+
+	while (mp) {
+		if (mp->report && !lws_metrics_check_in_policy(mp->report, name))
+			return mp;
+
+		mp = mp->next;
+	}
+
+	return NULL;
+}
+
+/*
+ * Create a lws_metric_t, bind to a named policy if possible (or add to the
+ * context list of unbound metrics) and set its lws_system
+ * idx.  The metrics objects themselves are typically composed into other
+ * objects and are well-known composed members of them.
+ */
+
+lws_metric_t *
+lws_metric_create(struct lws_context *ctx, uint8_t flags, const char *name)
+{
+	const lws_metric_policy_t *po;
+	lws_metric_policy_dyn_t *dmp;
+	lws_metric_pub_t *pub;
+	lws_metric_t *mt;
+	char pname[32];
+	size_t nl;
+
+	if (ctx->metrics_prefix) {
+
+		/*
+		 * In multi-process case, we want to prefix metrics from this
+		 * process / context with a string distinguishing which
+		 * application they came from
+		 */
+
+		nl = (size_t)lws_snprintf(pname, sizeof(pname) - 1, "%s.%s",
+				  ctx->metrics_prefix, name);
+		name = pname;
+	} else
+		nl = strlen(name);
+
+	mt = (lws_metric_t *)lws_zalloc(sizeof(*mt) /* private */ +
+					sizeof(lws_metric_pub_t) +
+					nl + 1 /* copy of metric name */,
+					__func__);
+	if (!mt)
+		return NULL;
+
+	pub = lws_metrics_priv_to_pub(mt);
+	pub->name = (char *)pub + sizeof(lws_metric_pub_t);
+	memcpy((char *)pub->name, name, nl + 1);
+	pub->flags = flags;
+
+	/* after these common members, we have to use the right type */
+
+	if (!(flags & LWSMTFL_REPORT_HIST)) {
+		/* anything is smaller or equal to this */
+		pub->u.agg.min = ~(u_mt_t)0;
+		pub->us_first = lws_now_usecs();
+	}
+
+	mt->ctx = ctx;
+
+	/*
+	 * Let's see if we can bind to a reporting policy straight away
+	 */
+
+	po = lws_metrics_find_policy(ctx, name);
+	if (po) {
+		dmp = lws_metrics_policy_get_dyn(ctx, po);
+		if (dmp) {
+			lwsl_notice("%s: metpol %s\n", __func__, name);
+			lws_dll2_add_tail(&mt->list, &dmp->owner);
+
+			return 0;
+		}
+	}
+
+	/*
+	 * If not, well, let's go on without and maybe later at runtime, he'll
+	 * get interested in us and apply a reporting policy
+	 */
+
+	lws_dll2_add_tail(&mt->list, &ctx->owner_mtr_no_pol);
+
+	return mt;
+}
+
+/*
+ * If our metric is bound to a reporting policy, return a pointer to it,
+ * otherwise NULL
+ */
+
+const lws_metric_policy_t *
+lws_metric_get_policy(lws_metric_t *mt)
+{
+	lws_metric_policy_dyn_t *dp;
+
+	/*
+	 * Our metric must either be on the "no policy" context list or
+	 * listed by the dynamic part of the policy it is bound to
+	 */
+	assert(mt->list.owner);
+
+	if ((char *)mt->list.owner >= (char *)mt->ctx &&
+	    (char *)mt->list.owner < (char *)mt->ctx + sizeof(struct lws_context))
+		/* we are on the "no policy" context list */
+		return NULL;
+
+	/* we are listed by a dynamic policy owner */
+
+	dp = lws_container_of(mt->list.owner, lws_metric_policy_dyn_t, owner);
+
+	/* return the const policy the dynamic policy represents */
+
+	return dp->policy;
+}
+
+void
+lws_metric_rebind_policies(struct lws_context *ctx)
+{
+	const lws_metric_policy_t *po;
+	lws_metric_policy_dyn_t *dmp;
+
+	lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1,
+				   ctx->owner_mtr_no_pol.head) {
+		lws_metric_t *mt = lws_container_of(d, lws_metric_t, list);
+		lws_metric_pub_t *pub = lws_metrics_priv_to_pub(mt);
+
+		po = lws_metrics_find_policy(ctx, pub->name);
+		if (po) {
+			dmp = lws_metrics_policy_get_dyn(ctx, po);
+			if (dmp) {
+				lwsl_info("%s: %s <- pol %s\n", __func__,
+						pub->name, po->name);
+				lws_dll2_remove(&mt->list);
+				lws_dll2_add_tail(&mt->list, &dmp->owner);
+			}
+		} else
+			lwsl_debug("%s: no pol for %s\n", __func__, pub->name);
+
+	} lws_end_foreach_dll_safe(d, d1);
+}
+
+int
+lws_metric_destroy(lws_metric_t **pmt, int keep)
+{
+	lws_metric_t *mt = *pmt;
+	lws_metric_pub_t *pub = lws_metrics_priv_to_pub(mt);
+
+	if (!mt)
+		return 0;
+
+	lws_dll2_remove(&mt->list);
+
+	if (keep) {
+		lws_dll2_add_tail(&mt->list, &mt->ctx->owner_mtr_no_pol);
+
+		return 0;
+	}
+
+	if (pub->flags & LWSMTFL_REPORT_HIST) {
+		lws_metric_bucket_t *b = pub->u.hist.head, *b1;
+
+		pub->u.hist.head = NULL;
+
+		while (b) {
+			b1 = b->next;
+			lws_free(b);
+			b = b1;
+		}
+	}
+
+	lws_free(mt);
+	*pmt = NULL;
+
+	return 0;
+}
+
+/*
+ * Allow an existing metric to have its reporting policy changed at runtime
+ */
+
+int
+lws_metric_switch_policy(lws_metric_t *mt, const char *polname)
+{
+	const lws_metric_policy_t *po;
+	lws_metric_policy_dyn_t *dmp;
+
+	po = lws_metrics_find_policy(mt->ctx, polname);
+	if (!po)
+		return 1;
+
+	dmp = lws_metrics_policy_get_dyn(mt->ctx, po);
+	if (!dmp)
+		return 1;
+
+	lws_dll2_remove(&mt->list);
+	lws_dll2_add_tail(&mt->list, &dmp->owner);
+
+	return 0;
+}
+
+/*
+ * If keep is set, don't destroy existing metrics objects, just detach them
+ * from the policy being deleted and keep track of them on ctx->
+ * owner_mtr_no_pol
+ */
+
+void
+lws_metric_policy_dyn_destroy(lws_metric_policy_dyn_t *dm, int keep)
+{
+	lws_sul_cancel(&dm->sul);
+
+	lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1, dm->owner.head) {
+		lws_metric_t *m = lws_container_of(d, lws_metric_t, list);
+
+		lws_metric_destroy(&m, keep);
+
+	} lws_end_foreach_dll_safe(d, d1);
+
+	lws_sul_cancel(&dm->sul);
+
+	lws_dll2_remove(&dm->list);
+	lws_free(dm);
+}
+
+/*
+ * Destroy all dynamic metrics policies, deinit any metrics still using them
+ */
+
+void
+lws_metrics_destroy(struct lws_context *ctx)
+{
+	lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1,
+				   ctx->owner_mtr_dynpol.head) {
+		lws_metric_policy_dyn_t *dm =
+			lws_container_of(d, lws_metric_policy_dyn_t, list);
+
+		lws_metric_policy_dyn_destroy(dm, 0); /* don't keep */
+
+	} lws_end_foreach_dll_safe(d, d1);
+
+	/* destroy metrics with no current policy too... */
+
+	lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1,
+				   ctx->owner_mtr_no_pol.head) {
+		lws_metric_t *mt = lws_container_of(d, lws_metric_t, list);
+
+		lws_metric_destroy(&mt, 0); /* don't keep */
+
+	} lws_end_foreach_dll_safe(d, d1);
+
+	/* ... that's the whole allocated metrics footprint gone... */
+}
+
+int
+lws_metrics_hist_bump_(lws_metric_pub_t *pub, const char *name)
+{
+	lws_metric_bucket_t *buck = pub->u.hist.head;
+	size_t nl = strlen(name);
+	char *nm;
+
+	if (!(pub->flags & LWSMTFL_REPORT_HIST)) {
+		lwsl_err("%s: %s not histogram: flags %d\n", __func__,
+				pub->name, pub->flags);
+		assert(0);
+	}
+	assert(nl < 255);
+
+	pub->us_last = lws_now_usecs();
+	if (!pub->us_first)
+		pub->us_first = pub->us_last;
+
+	while (buck) {
+		if (lws_metric_bucket_name_len(buck) == nl &&
+		    !strcmp(name, lws_metric_bucket_name(buck))) {
+			buck->count++;
+			goto happy;
+		}
+		buck = buck->next;
+	}
+
+	buck = lws_malloc(sizeof(*buck) + nl + 2, __func__);
+	if (!buck)
+		return 1;
+
+	nm = (char *)buck + sizeof(*buck);
+	/* length byte at beginning of name, avoid struct alignment overhead */
+	*nm = (char)nl;
+	memcpy(nm + 1, name, nl + 1);
+
+	buck->next = pub->u.hist.head;
+	pub->u.hist.head = buck;
+	buck->count = 1;
+	pub->u.hist.list_size++;
+
+happy:
+	pub->u.hist.total_count++;
+
+	return 0;
+}
+
+int
+lws_metrics_hist_bump_describe_wsi(struct lws *wsi, lws_metric_pub_t *pub,
+				   const char *name)
+{
+	char desc[192], d1[48], *p = desc, *end = desc + sizeof(desc);
+
+#if defined(LWS_WITH_SECURE_STREAMS)
+#if defined(LWS_WITH_SECURE_STREAMS_PROXY_API)
+	if (wsi->client_bound_sspc) {
+		lws_sspc_handle_t *h = (lws_sspc_handle_t *)wsi->a.opaque_user_data;
+		if (h)
+			p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), "ss=\"%s\",",
+				  h->ssi.streamtype);
+	} else
+		if (wsi->client_proxy_onward) {
+			struct conn *conn = (struct conn *)wsi->a.opaque_user_data;
+
+			if (conn && conn->ss)
+			p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), "ss=\"%s\",",
+				  conn->ss->info.streamtype);
+		} else
+#endif
+	if (wsi->for_ss) {
+		lws_ss_handle_t *h = (lws_ss_handle_t *)wsi->a.opaque_user_data;
+		if (h)
+			p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), "ss=\"%s\",",
+				  h->info.streamtype);
+	}
+#endif
+
+#if defined(LWS_WITH_CLIENT)
+	if (wsi->stash && wsi->stash->cis[CIS_HOST])
+		p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), "hostname=\"%s\",",
+				wsi->stash->cis[CIS_HOST]);
+#endif
+
+	lws_sa46_write_numeric_address(&wsi->sa46_peer, d1, sizeof(d1));
+	p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), "peer=\"%s\",", d1);
+
+	p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), "%s", name);
+
+	lws_metrics_hist_bump_(pub, desc);
+
+	return 0;
+}
+
+int
+lws_metrics_foreach(struct lws_context *ctx, void *user,
+		    int (*cb)(lws_metric_pub_t *pub, void *user))
+{
+	int n;
+
+	lws_start_foreach_dll_safe(struct lws_dll2 *, d, d1,
+				   ctx->owner_mtr_no_pol.head) {
+		lws_metric_t *mt = lws_container_of(d, lws_metric_t, list);
+
+		n = cb(lws_metrics_priv_to_pub(mt), user);
+		if (n)
+			return n;
+
+	} lws_end_foreach_dll_safe(d, d1);
+
+	lws_start_foreach_dll_safe(struct lws_dll2 *, d2, d3,
+				   ctx->owner_mtr_dynpol.head) {
+		lws_metric_policy_dyn_t *dm =
+			lws_container_of(d2, lws_metric_policy_dyn_t, list);
+
+		lws_start_foreach_dll_safe(struct lws_dll2 *, e, e1,
+					   dm->owner.head) {
+
+			lws_metric_t *mt = lws_container_of(e, lws_metric_t, list);
+
+			n = cb(lws_metrics_priv_to_pub(mt), user);
+			if (n)
+				return n;
+
+		} lws_end_foreach_dll_safe(e, e1);
+
+	} lws_end_foreach_dll_safe(d2, d3);
+
+	return 0;
+}
+
+static int
+lws_metrics_dump_cb(lws_metric_pub_t *pub, void *user)
+{
+	struct lws_context *ctx = (struct lws_context *)user;
+	int n;
+
+	if (!ctx->system_ops || !ctx->system_ops->metric_report)
+		return 0;
+
+	/*
+	 * return nonzero to reset stats
+	 */
+
+	n = ctx->system_ops->metric_report(pub);
+
+	/* track when we dumped it... */
+
+	pub->us_first = pub->us_dumped = lws_now_usecs();
+	pub->us_last = 0;
+
+	if (!n)
+		return 0;
+
+	/* ... and clear it back to 0 */
+
+	if (pub->flags & LWSMTFL_REPORT_HIST) {
+		lws_metric_bucket_t *b = pub->u.hist.head, *b1;
+		pub->u.hist.head = NULL;
+
+		while (b) {
+			b1 = b->next;
+			lws_free(b);
+			b = b1;
+		}
+		pub->u.hist.total_count = 0;
+		pub->u.hist.list_size = 0;
+	} else
+		memset(&pub->u.agg, 0, sizeof(pub->u.agg));
+
+	return 0;
+}
+
+void
+lws_metrics_dump(struct lws_context *ctx)
+{
+	lws_metrics_foreach(ctx, ctx, lws_metrics_dump_cb);
+}
+
+static int
+_lws_metrics_format(lws_metric_pub_t *pub, lws_usec_t now, int gng,
+		    char *buf, size_t len)
+{
+	const lws_humanize_unit_t *schema = humanize_schema_si;
+	char *end = buf + len - 1, *obuf = buf;
+
+	if (pub->flags & LWSMTFL_REPORT_DUTY_WALLCLOCK_US)
+		schema = humanize_schema_us;
+
+	if (!(pub->flags & LWSMTFL_REPORT_MEAN)) {
+		/* only the sum is meaningful */
+		if (pub->flags & LWSMTFL_REPORT_DUTY_WALLCLOCK_US) {
+
+			buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), " %u, ",
+						(unsigned int)pub->u.agg.count[gng]);
+
+			buf += lws_humanize(buf, lws_ptr_diff_size_t(end, buf),
+					    (uint64_t)pub->u.agg.sum[gng],
+					    humanize_schema_us);
+
+			buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), " / ");
+
+			buf += lws_humanize(buf, lws_ptr_diff_size_t(end, buf),
+					    (uint64_t)(now - pub->us_first),
+					    humanize_schema_us);
+
+			buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf),
+					    " (%d%%)", (int)((100 * pub->u.agg.sum[gng]) /
+						(unsigned long)(now - pub->us_first)));
+		} else {
+			/* it's a monotonic ordinal, like total tx */
+			buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), "(%u) ",
+					(unsigned int)pub->u.agg.count[gng]);
+			buf += lws_humanize(buf, lws_ptr_diff_size_t(end, buf),
+					    (uint64_t)pub->u.agg.sum[gng],
+					    humanize_schema_si);
+		}
+
+	} else {
+		buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), "%u, mean: ", (unsigned int)pub->u.agg.count[gng]);
+		/* the average over the period is meaningful */
+		buf += lws_humanize(buf, lws_ptr_diff_size_t(end, buf),
+				    (uint64_t)(pub->u.agg.count[gng] ?
+					 pub->u.agg.sum[gng] / pub->u.agg.count[gng] : 0),
+				    schema);
+	}
+
+	return lws_ptr_diff(buf, obuf);
+}
+
+int
+lws_metrics_format(lws_metric_pub_t *pub, lws_metric_bucket_t **sub, char *buf, size_t len)
+{
+	char *end = buf + len - 1, *obuf = buf;
+	lws_usec_t t = lws_now_usecs();
+	const lws_humanize_unit_t *schema = humanize_schema_si;
+
+	if (pub->flags & LWSMTFL_REPORT_DUTY_WALLCLOCK_US)
+		schema = humanize_schema_us;
+
+	if (pub->flags & LWSMTFL_REPORT_HIST) {
+
+		if (*sub == NULL)
+			return 0;
+
+		if (*sub) {
+			buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf),
+					    "%s{%s} %llu", pub->name,
+					    lws_metric_bucket_name(*sub),
+					    (unsigned long long)(*sub)->count);
+
+			*sub = (*sub)->next;
+		}
+
+		goto happy;
+	}
+
+	buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), "%s: ",
+				pub->name);
+
+	if (!pub->u.agg.count[METRES_GO] && !pub->u.agg.count[METRES_NOGO])
+		return 0;
+
+	if (pub->u.agg.count[METRES_GO]) {
+		if (!(pub->flags & LWSMTFL_REPORT_ONLY_GO))
+			buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf),
+					    "Go: ");
+		buf += _lws_metrics_format(pub, t, METRES_GO, buf,
+					   lws_ptr_diff_size_t(end, buf));
+	}
+
+	if (!(pub->flags & LWSMTFL_REPORT_ONLY_GO) && pub->u.agg.count[METRES_NOGO]) {
+		buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), ", NoGo: ");
+		buf += _lws_metrics_format(pub, t, METRES_NOGO, buf,
+					   lws_ptr_diff_size_t(end, buf));
+	}
+
+	if (pub->flags & LWSMTFL_REPORT_MEAN) {
+		buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), ", min: ");
+		buf += lws_humanize(buf, lws_ptr_diff_size_t(end, buf), pub->u.agg.min,
+				    schema);
+		buf += lws_snprintf(buf, lws_ptr_diff_size_t(end, buf), ", max: ");
+		buf += lws_humanize(buf, lws_ptr_diff_size_t(end, buf), pub->u.agg.max,
+				    schema);
+	}
+
+happy:
+	if (pub->flags & LWSMTFL_REPORT_HIST)
+		return 1;
+
+	*sub = NULL;
+
+	return lws_ptr_diff(buf, obuf);
+}
+
+/*
+ * We want to, at least internally, record an event... depending on the policy,
+ * that might cause us to call through to the lws_system apis, or just update
+ * our local stats about it and dump at the next periodic chance (also set by
+ * the policy)
+ */
+
+void
+lws_metric_event(lws_metric_t *mt, char go_nogo, u_mt_t val)
+{
+	lws_metric_pub_t *pub;
+
+	assert((go_nogo & 0xfe) == 0);
+
+	if (!mt)
+		return;
+
+	pub = lws_metrics_priv_to_pub(mt);
+	assert(!(pub->flags & LWSMTFL_REPORT_HIST));
+
+	pub->us_last = lws_now_usecs();
+	if (!pub->us_first)
+		pub->us_first = pub->us_last;
+	pub->u.agg.count[(int)go_nogo]++;
+	pub->u.agg.sum[(int)go_nogo] += val;
+	if (val > pub->u.agg.max)
+		pub->u.agg.max = val;
+	if (val < pub->u.agg.min)
+		pub->u.agg.min = val;
+
+	if (pub->flags & LWSMTFL_REPORT_OOB)
+		lws_metrics_report_and_maybe_clear(mt->ctx, pub);
+}
+
+
+void
+lws_metrics_hist_bump_priv_tagged(lws_metric_pub_t *mt, lws_dll2_owner_t *tow,
+				  lws_dll2_owner_t *tow2)
+{
+	char qual[192];
+	size_t p;
+
+	p = lws_metrics_tags_serialize(tow, qual, sizeof(qual));
+	if (tow2)
+		lws_metrics_tags_serialize(tow2, qual + p,
+				sizeof(qual) - p);
+
+	lwsl_warn("%s: '%s'\n", __func__, qual);
+
+	lws_metrics_hist_bump(mt, qual);
+}

lib/system/metrics/private-lib-system-metrics.h

@@ -0,0 +1,124 @@
+/*
+ * lws System Metrics
+ *
+ * Copyright (C) 2021 Andy Green <andy@warmcat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+/*
+ * Const struct that describes a policy for processing raw metrics to turn them
+ * into events.
+ *
+ * Typically although we want to monitor every event, the data produced can be
+ * too large, and many events that are "normal" just need to be counted as such;
+ * outliers or change-to-continuous outliers may deserve closer recording as
+ * events in their own right.
+ *
+ * Mean computation must "decay" as it ages, we do this by halving the sum and
+ * count after .us_decay_unit us.
+ *
+ * We don't acknowledge outliers until there are at least .min_contributors
+ * in the current mean (which is subject to decaying)
+ *
+ * We decide something is an outlier event if it deviates from the mean by
+ * .pc_outlier_deviation %.
+ */
+
+/*
+ * The dynamic counterpart for each static metric policy, this is on heap
+ * one per const lws_metric_policy_t.  It's listed in context->owner_mtr_dynpol
+ */
+
+typedef struct lws_metric_policy_dyn {
+	const lws_metric_policy_t	*policy;
+	/**< the static part of the policy we belong to... can be NULL if no
+	 * policy matches or the policy was invalidated */
+
+	lws_dll2_owner_t		owner;
+	/**< list of metrics that are using this policy */
+
+	lws_dll2_t			list;
+	/**< context owns us */
+
+	lws_sorted_usec_list_t		sul;
+	/**< schedule periodic reports for metrics using this policy */
+} lws_metric_policy_dyn_t;
+
+/*
+ * A metrics private part, encapsulating the public part
+ */
+
+typedef struct lws_metric {
+
+	lws_dll2_t			list;
+	/**< owned by either 1) ctx.lws_metric_policy_dyn_t.owner, or
+	 * 2) ctx.owner_mtr_no_pol */
+
+	struct lws_context		*ctx;
+
+	/* public part overallocated */
+} lws_metric_t;
+
+
+#if defined(LWS_WITH_SYS_METRICS)
+#define lws_metrics_hist_bump_priv(_mt, _name) \
+		lws_metrics_hist_bump_(lws_metrics_priv_to_pub(_mt), _name)
+#define lws_metrics_hist_bump_priv_wsi(_wsi, _hist, _name) \
+		lws_metrics_hist_bump_(lws_metrics_priv_to_pub(_wsi->a.context->_hist), _name)
+#define lws_metrics_hist_bump_priv_ss(_ss, _hist, _name) \
+		lws_metrics_hist_bump_(lws_metrics_priv_to_pub(_ss->context->_hist), _name)
+#define lws_metrics_priv_to_pub(_x) ((lws_metric_pub_t *)&(_x)[1])
+#else
+#define lws_metrics_hist_bump_priv(_mt, _name)
+#define lws_metrics_hist_bump_priv_wsi(_wsi, _hist, _name)
+#define lws_metrics_hist_bump_priv_ss(_ss, _hist, _name)
+#define lws_metrics_priv_to_pub(_x) ((lws_metric_pub_t *)NULL)
+#endif
+
+#if defined(LWS_WITH_SECURE_STREAMS_PROXY_API)
+/*
+ * sspc-specific version that also appends the tag value to the lifecycle tag
+ * used for logging the sspc identity
+ */
+int
+lws_metrics_tag_sspc_add(struct lws_sspc_handle *ss, const char *name, const char *val);
+#endif
+
+int
+lws_metrics_register_policy(struct lws_context *ctx,
+			    const lws_metric_policy_t *head);
+
+void
+lws_metrics_destroy(struct lws_context *ctx);
+
+void
+lws_metric_event(lws_metric_t *mt, char go_nogo, u_mt_t val);
+
+lws_metric_t *
+lws_metric_create(struct lws_context *ctx, uint8_t flags, const char *name);
+
+int
+lws_metric_destroy(lws_metric_t **mt, int keep);
+
+void
+lws_metric_policy_dyn_destroy(lws_metric_policy_dyn_t *dm, int keep);
+
+void
+lws_metric_rebind_policies(struct lws_context *ctx);

lib/system/smd/smd.c

@@ -377,6 +377,10 @@ _lws_smd_ss_rx_forward(struct lws_context *ctx, const char *tag,
 
 	_class = (lws_smd_class_t)lws_ser_ru64be(buf);
 
+	if (_class == LWSSMDCL_METRICS) {
+
+	}
+
 	/* only locally forward messages that we care about in this process */
 
 	if (!(ctx->smd._class_filter & _class))

lib/tls/mbedtls/mbedtls-client.c

@@ -223,47 +223,75 @@ int
 lws_tls_client_confirm_peer_cert(struct lws *wsi, char *ebuf, size_t ebuf_len)
 {
 	int n;
+	unsigned int avoid = 0;
 	X509 *peer = SSL_get_peer_certificate(wsi->tls.ssl);
 	struct lws_context_per_thread *pt = &wsi->a.context->pt[(int)wsi->tsi];
+	const char *type = "";
 	char *sb = (char *)&pt->serv_buf[0];
 
 	if (!peer) {
+#if defined(LWS_WITH_SYS_METRICS)
+		lws_metrics_hist_bump_describe_wsi(wsi, lws_metrics_priv_to_pub(
+					wsi->a.context->mth_conn_failures),
+						   "tls=\"nocert\"");
+#endif
 		lwsl_info("peer did not provide cert\n");
 		lws_snprintf(ebuf, ebuf_len, "no peer cert");
 
 		return -1;
 	}
-	lwsl_info("peer provided cert\n");
 
 	n = (int)SSL_get_verify_result(wsi->tls.ssl);
-        lwsl_debug("get_verify says %d\n", n);
+	lwsl_debug("get_verify says %d\n", n);
 
-	if (n == X509_V_OK)
+	switch (n) {
+	case X509_V_OK:
 		return 0;
 
-	if (n == X509_V_ERR_HOSTNAME_MISMATCH &&
+	case X509_V_ERR_HOSTNAME_MISMATCH:
-	    (wsi->tls.use_ssl & LCCSCF_SKIP_SERVER_CERT_HOSTNAME_CHECK)) {
+		type = "hostname";
-		lwsl_info("accepting certificate for invalid hostname\n");
+		avoid = LCCSCF_SKIP_SERVER_CERT_HOSTNAME_CHECK;
+		break;
+
+	case X509_V_ERR_INVALID_CA:
+		type = "invalidca";
+		avoid = LCCSCF_ALLOW_SELFSIGNED;
+		break;
+
+	case X509_V_ERR_CERT_NOT_YET_VALID:
+		type = "notyetvalid";
+		avoid = LCCSCF_ALLOW_EXPIRED;
+		break;
+
+	case X509_V_ERR_CERT_HAS_EXPIRED:
+		type = "expired";
+		avoid = LCCSCF_ALLOW_EXPIRED;
+		break;
+	}
+
+	lwsl_info("%s: cert problem: %s\n", __func__, type);
+#if defined(LWS_WITH_SYS_METRICS)
+	{
+		char buckname[64];
+		lws_snprintf(buckname, sizeof(buckname), "tls=\"%s\"", type);
+		lws_metrics_hist_bump_describe_wsi(wsi,
+		     lws_metrics_priv_to_pub(wsi->a.context->mth_conn_failures),
+			      buckname);
+	}
+#endif
+	if (wsi->tls.use_ssl & avoid) {
+		lwsl_info("%s: allowing anyway\n", __func__);
+
 		return 0;
 	}
 
-	if (n == X509_V_ERR_INVALID_CA &&
-	    (wsi->tls.use_ssl & LCCSCF_ALLOW_SELFSIGNED)) {
-		lwsl_info("accepting certificate from untrusted CA\n");
-		return 0;
-	}
-
-	if ((n == X509_V_ERR_CERT_NOT_YET_VALID ||
-	     n == X509_V_ERR_CERT_HAS_EXPIRED) &&
-	     (wsi->tls.use_ssl & LCCSCF_ALLOW_EXPIRED)) {
-		lwsl_info("accepting expired or not yet valid certificate\n");
-
-		return 0;
-	}
 	lws_snprintf(ebuf, ebuf_len,
-		"server's cert didn't look good, (use_ssl 0x%x) X509_V_ERR = %d: %s\n",
+		"server's cert didn't look good, %s (use_ssl 0x%x) X509_V_ERR = %d: %s\n",
-		(unsigned int)wsi->tls.use_ssl, n, ERR_error_string((unsigned long)n, sb));
+		type, (unsigned int)wsi->tls.use_ssl, n,
+		ERR_error_string((unsigned long)n, sb));
+
 	lwsl_info("%s\n", ebuf);
+
 	lws_tls_err_describe_clear();
 
 	return -1;

lib/tls/mbedtls/mbedtls-ssl.c

@@ -51,8 +51,6 @@ lws_ssl_capable_read(struct lws *wsi, unsigned char *buf, size_t len)
 	if (!wsi->tls.ssl)
 		return lws_ssl_capable_read_no_ssl(wsi, buf, len);
 
-	lws_stats_bump(pt, LWSSTATS_C_API_READ, 1);
-
 	errno = 0;
 	n = SSL_read(wsi->tls.ssl, buf, (int)len);
 #if defined(LWS_PLAT_FREERTOS)
@@ -61,15 +59,6 @@ lws_ssl_capable_read(struct lws *wsi, unsigned char *buf, size_t len)
 		return LWS_SSL_CAPABLE_ERROR;
 	}
 #endif
-#if defined(LWS_WITH_STATS)
-	if (!wsi->seen_rx && wsi->accept_start_us) {
-                lws_stats_bump(pt, LWSSTATS_US_SSL_RX_DELAY_AVG,
-			lws_now_usecs() - wsi->accept_start_us);
-                lws_stats_bump(pt, LWSSTATS_C_SSL_CONNS_HAD_RX, 1);
-		wsi->seen_rx = 1;
-	}
-#endif
-
 
 	lwsl_debug("%s: %s: SSL_read says %d\n", __func__, lws_wsi_tag(wsi), n);
 	/* manpage: returning 0 means connection shut down */
@@ -82,14 +71,13 @@ lws_ssl_capable_read(struct lws *wsi, unsigned char *buf, size_t len)
 	if (n < 0) {
 		m = SSL_get_error(wsi->tls.ssl, n);
 		lwsl_debug("%s: %s: ssl err %d errno %d\n", __func__, lws_wsi_tag(wsi), m, errno);
-		if (errno == LWS_ENOTCONN) {
+		if (errno == LWS_ENOTCONN)
 			/* If the socket isn't connected anymore, bail out. */
-			wsi->socket_is_permanently_unusable = 1;
+			goto do_err1;
-			return LWS_SSL_CAPABLE_ERROR;
+
-		}
 		if (m == SSL_ERROR_ZERO_RETURN ||
 		    m == SSL_ERROR_SYSCALL)
-			return LWS_SSL_CAPABLE_ERROR;
+			goto do_err;
 
 		if (m == SSL_ERROR_WANT_READ || SSL_want_read(wsi->tls.ssl)) {
 			lwsl_debug("%s: WANT_READ\n", __func__);
@@ -101,8 +89,16 @@ lws_ssl_capable_read(struct lws *wsi, unsigned char *buf, size_t len)
 			lwsl_debug("%s: LWS_SSL_CAPABLE_MORE_SERVICE\n", lws_wsi_tag(wsi));
 			return LWS_SSL_CAPABLE_MORE_SERVICE;
 		}
+
+do_err1:
 		wsi->socket_is_permanently_unusable = 1;
 
+do_err:
+#if defined(LWS_WITH_SYS_METRICS)
+	if (wsi->a.vhost)
+		lws_metric_event(wsi->a.vhost->mt_traffic_rx, METRES_NOGO, 0);
+#endif
+
 		return LWS_SSL_CAPABLE_ERROR;
 	}
 
@@ -115,23 +111,12 @@ lws_ssl_capable_read(struct lws *wsi, unsigned char *buf, size_t len)
 	lwsl_hexdump_notice(buf, n);
 #endif
 
-	lws_stats_bump(pt, LWSSTATS_B_READ, (uint64_t)n);
+#if defined(LWS_WITH_SYS_METRICS)
-
-#if defined(LWS_WITH_SERVER_STATUS)
 	if (wsi->a.vhost)
-		wsi->a.vhost->conn_stats.rx = wsi->a.vhost->conn_stats.rx + (unsigned long long)n;
+		lws_metric_event(wsi->a.vhost->mt_traffic_rx,
-#endif
+				 METRES_GO /* rx */, (u_mt_t)n);
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	if (context->detailed_latency_cb) {
-		wsi->detlat.req_size = len;
-		wsi->detlat.acc_size = n;
-		wsi->detlat.type = LDLT_READ;
-		wsi->detlat.latencies[LAT_DUR_PROXY_RX_TO_ONWARD_TX] =
-			lws_now_usecs() - pt->ust_left_poll;
-		wsi->detlat.latencies[LAT_DUR_USERCB] = 0;
-		lws_det_lat_cb(wsi->a.context, &wsi->detlat);
-	}
 #endif
+
 	/*
 	 * if it was our buffer that limited what we read,
 	 * check if SSL has additional data pending inside SSL buffers.
@@ -186,8 +171,14 @@ lws_ssl_capable_write(struct lws *wsi, unsigned char *buf, size_t len)
 		return lws_ssl_capable_write_no_ssl(wsi, buf, len);
 
 	n = SSL_write(wsi->tls.ssl, buf, (int)len);
-	if (n > 0)
+	if (n > 0) {
+#if defined(LWS_WITH_SYS_METRICS)
+		if (wsi->a.vhost)
+			lws_metric_event(wsi->a.vhost->mt_traffic_tx,
+					 METRES_GO, (u_mt_t)n);
+#endif
 		return n;
+	}
 
 	m = SSL_get_error(wsi->tls.ssl, n);
 	if (m != SSL_ERROR_SYSCALL) {
@@ -208,6 +199,12 @@ lws_ssl_capable_write(struct lws *wsi, unsigned char *buf, size_t len)
 	lwsl_debug("%s failed: %d\n",__func__, m);
 	wsi->socket_is_permanently_unusable = 1;
 
+#if defined(LWS_WITH_SYS_METRICS)
+		if (wsi->a.vhost)
+			lws_metric_event(wsi->a.vhost->mt_traffic_tx,
+					 METRES_NOGO, (u_mt_t)n);
+#endif
+
 	return LWS_SSL_CAPABLE_ERROR;
 }
 

lib/tls/openssl/openssl-client.c

@@ -125,6 +125,18 @@ OpenSSL_client_verify_callback(int preverify_ok, X509_STORE_CTX *x509_ctx)
 			lwsl_err("SSL error: %s (preverify_ok=%d;err=%d;"
 				 "depth=%d)\n", msg, preverify_ok, err, depth);
 
+#if defined(LWS_WITH_SYS_METRICS)
+			{
+				char buckname[64];
+
+				lws_snprintf(buckname, sizeof(buckname),
+					     "tls=\"%s\"", msg);
+				lws_metrics_hist_bump_describe_wsi(wsi,
+					lws_metrics_priv_to_pub(wsi->a.context->mth_conn_failures),
+					buckname);
+			}
+#endif
+
 			return preverify_ok;	// not ok
 		}
 	}
@@ -480,7 +492,8 @@ lws_tls_client_confirm_peer_cert(struct lws *wsi, char *ebuf, size_t ebuf_len)
 #if !defined(USE_WOLFSSL)
 	struct lws_context_per_thread *pt = &wsi->a.context->pt[(int)wsi->tsi];
 	char *p = (char *)&pt->serv_buf[0];
-	const char *es;
+	const char *es, *type = "";
+	unsigned int avoid = 0;
 	char *sb = p;
 	long n;
 
@@ -488,29 +501,46 @@ lws_tls_client_confirm_peer_cert(struct lws *wsi, char *ebuf, size_t ebuf_len)
 	ERR_clear_error();
 	n = SSL_get_verify_result(wsi->tls.ssl);
 
-	lwsl_debug("get_verify says %ld\n", n);
+	switch (n) {
-
+	case X509_V_OK:
-	if (n == X509_V_OK)
 		return 0;
 
-	if ((n == X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT ||
+	case X509_V_ERR_HOSTNAME_MISMATCH:
-	     n == X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN) &&
+		type = "tls=hostname";
-	     (wsi->tls.use_ssl & LCCSCF_ALLOW_SELFSIGNED)) {
+		avoid = LCCSCF_SKIP_SERVER_CERT_HOSTNAME_CHECK;
-		lwsl_info("accepting self-signed certificate\n");
+		break;
+
+	case X509_V_ERR_INVALID_CA:
+	case X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT:
+	case X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN:
+		type = "tls=invalidca";
+		avoid = LCCSCF_ALLOW_SELFSIGNED;
+		break;
+
+	case X509_V_ERR_CERT_NOT_YET_VALID:
+		type = "tls=notyetvalid";
+		avoid = LCCSCF_ALLOW_EXPIRED;
+		break;
+
+	case X509_V_ERR_CERT_HAS_EXPIRED:
+		type = "tls=expired";
+		avoid = LCCSCF_ALLOW_EXPIRED;
+		break;
+	}
+
+	lwsl_info("%s: cert problem: %s\n", __func__, type);
+
+#if defined(LWS_WITH_SYS_METRICS)
+	lws_metrics_hist_bump_describe_wsi(wsi,
+			lws_metrics_priv_to_pub(wsi->a.context->mth_conn_failures), type);
+#endif
+
+	if (wsi->tls.use_ssl & avoid) {
+		lwsl_info("%s: allowing anyway\n", __func__);
 
 		return 0;
 	}
-	if ((n == X509_V_ERR_CERT_NOT_YET_VALID ||
+
-	     n == X509_V_ERR_CERT_HAS_EXPIRED) &&
-	     (wsi->tls.use_ssl & LCCSCF_ALLOW_EXPIRED)) {
-		lwsl_info("accepting expired certificate\n");
-		return 0;
-	}
-	if (n == X509_V_ERR_CERT_NOT_YET_VALID) {
-		lwsl_info("Cert is from the future... "
-			    "probably our clock... accepting...\n");
-		return 0;
-	}
 	es = ERR_error_string(
 	#if defined(LWS_WITH_BORINGSSL)
 					 (uint32_t)
@@ -519,8 +549,8 @@ lws_tls_client_confirm_peer_cert(struct lws *wsi, char *ebuf, size_t ebuf_len)
 	#endif
 					 n, sb);
 	lws_snprintf(ebuf, ebuf_len,
-		"server's cert didn't look good, X509_V_ERR = %ld: %s\n",
+		"server's cert didn't look good, %s X509_V_ERR = %ld: %s\n",
-		 n, es);
+		 type, n, es);
 	lwsl_info("%s\n", ebuf);
 	lws_tls_err_describe_clear();
 

lib/tls/openssl/openssl-ssl.c

@@ -207,8 +207,6 @@ lws_ssl_capable_read(struct lws *wsi, unsigned char *buf, size_t len)
 	if (!wsi->tls.ssl)
 		return lws_ssl_capable_read_no_ssl(wsi, buf, len);
 
-	lws_stats_bump(pt, LWSSTATS_C_API_READ, 1);
-
 	errno = 0;
 	ERR_clear_error();
 	n = SSL_read(wsi->tls.ssl, buf, (int)(ssize_t)len);
@@ -218,16 +216,6 @@ lws_ssl_capable_read(struct lws *wsi, unsigned char *buf, size_t len)
 		return LWS_SSL_CAPABLE_ERROR;
 	}
 #endif
-#if defined(LWS_WITH_STATS)
-	if (!wsi->seen_rx && wsi->accept_start_us) {
-                lws_stats_bump(pt, LWSSTATS_US_SSL_RX_DELAY_AVG,
-                		      lws_now_usecs() -
-                			      wsi->accept_start_us);
-                lws_stats_bump(pt, LWSSTATS_C_SSL_CONNS_HAD_RX, 1);
-		wsi->seen_rx = 1;
-	}
-#endif
-
 
 	lwsl_debug("%s: SSL_read says %d\n", lws_wsi_tag(wsi), n);
 	/* manpage: returning 0 means connection shut down
@@ -258,7 +246,7 @@ lws_ssl_capable_read(struct lws *wsi, unsigned char *buf, size_t len)
 		m = lws_ssl_get_error(wsi, n);
 		lwsl_debug("%s: ssl err %d errno %d\n", lws_wsi_tag(wsi), m, errno);
 		if (m == SSL_ERROR_ZERO_RETURN) /* cleanly shut down */
-			return LWS_SSL_CAPABLE_ERROR;
+			goto do_err;
 
 		/* hm not retryable.. could be 0 size pkt or error  */
 
@@ -268,7 +256,12 @@ lws_ssl_capable_read(struct lws *wsi, unsigned char *buf, size_t len)
 			/* unclean, eg closed conn */
 
 			wsi->socket_is_permanently_unusable = 1;
-
+do_err:
+#if defined(LWS_WITH_SYS_METRICS)
+		if (wsi->a.vhost)
+			lws_metric_event(wsi->a.vhost->mt_traffic_rx,
+					 METRES_NOGO, 0);
+#endif
 			return LWS_SSL_CAPABLE_ERROR;
 		}
 
@@ -294,26 +287,12 @@ lws_ssl_capable_read(struct lws *wsi, unsigned char *buf, size_t len)
 	 * paths to dump what was received as decrypted data from the tls tunnel
 	 */
 	lwsl_notice("%s: len %d\n", __func__, n);
-	lwsl_hexdump_notice(buf, n);
+	lwsl_hexdump_notice(buf, (unsigned int)n);
 #endif
 
-	lws_stats_bump(pt, LWSSTATS_B_READ, (unsigned int)n);
+#if defined(LWS_WITH_SYS_METRICS)
-
-#if defined(LWS_WITH_SERVER_STATUS)
 	if (wsi->a.vhost)
-		wsi->a.vhost->conn_stats.rx = (unsigned long long)(wsi->a.vhost->conn_stats.rx + (unsigned long long)(long long)n);
+		lws_metric_event(wsi->a.vhost->mt_traffic_rx, METRES_GO, (u_mt_t)n);
-#endif
-
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	if (context->detailed_latency_cb) {
-		wsi->detlat.req_size = len;
-		wsi->detlat.acc_size = (unsigned int)n;
-		wsi->detlat.type = LDLT_READ;
-		wsi->detlat.latencies[LAT_DUR_PROXY_RX_TO_ONWARD_TX] =
-			(uint32_t)(lws_now_usecs() - pt->ust_left_poll);
-		wsi->detlat.latencies[LAT_DUR_USERCB] = 0;
-		lws_det_lat_cb(wsi->a.context, &wsi->detlat);
-	}
 #endif
 
 	/*
@@ -363,7 +342,7 @@ lws_ssl_capable_write(struct lws *wsi, unsigned char *buf, size_t len)
 	 * paths before sending data into the tls tunnel, where you can dump it
 	 * and see what is being sent.
 	 */
-	lwsl_notice("%s: len %d\n", __func__, len);
+	lwsl_notice("%s: len %u\n", __func__, (unsigned int)len);
 	lwsl_hexdump_notice(buf, len);
 #endif
 
@@ -373,8 +352,14 @@ lws_ssl_capable_write(struct lws *wsi, unsigned char *buf, size_t len)
 	errno = 0;
 	ERR_clear_error();
 	n = SSL_write(wsi->tls.ssl, buf, (int)(ssize_t)len);
-	if (n > 0)
+	if (n > 0) {
+#if defined(LWS_WITH_SYS_METRICS)
+		if (wsi->a.vhost)
+			lws_metric_event(wsi->a.vhost->mt_traffic_tx,
+					 METRES_GO, (u_mt_t)n);
+#endif
 		return n;
+	}
 
 	m = lws_ssl_get_error(wsi, n);
 	if (m != SSL_ERROR_SYSCALL) {
@@ -398,6 +383,12 @@ lws_ssl_capable_write(struct lws *wsi, unsigned char *buf, size_t len)
 
 	wsi->socket_is_permanently_unusable = 1;
 
+#if defined(LWS_WITH_SYS_METRICS)
+		if (wsi->a.vhost)
+			lws_metric_event(wsi->a.vhost->mt_traffic_tx,
+					 METRES_NOGO, 0);
+#endif
+
 	return LWS_SSL_CAPABLE_ERROR;
 }
 

lib/tls/tls-client.c

@@ -34,6 +34,7 @@ lws_ssl_client_connect1(struct lws *wsi, char *errbuf, size_t len)
 	case LWS_SSL_CAPABLE_ERROR:
 		return -1;
 	case LWS_SSL_CAPABLE_DONE:
+		lws_metrics_caliper_report(wsi->cal_conn, METRES_GO);
 		return 1; /* connected */
 	case LWS_SSL_CAPABLE_MORE_SERVICE_WRITE:
 		lws_callback_on_writable(wsi);
@@ -73,8 +74,12 @@ lws_ssl_client_connect2(struct lws *wsi, char *errbuf, size_t len)
 		}
 	}
 
-	if (lws_tls_client_confirm_peer_cert(wsi, errbuf, len))
+	if (lws_tls_client_confirm_peer_cert(wsi, errbuf, len)) {
+		lws_metrics_caliper_report(wsi->cal_conn, METRES_NOGO);
 		return -1;
+	}
+
+	lws_metrics_caliper_report(wsi->cal_conn, METRES_GO);
 
 	return 1;
 }
@@ -187,6 +192,9 @@ lws_client_create_tls(struct lws *wsi, const char **pcce, int do_c1)
 		if (!do_c1)
 			return 0;
 
+		lws_metrics_caliper_report(wsi->cal_conn, METRES_GO);
+		lws_metrics_caliper_bind(wsi->cal_conn, wsi->a.context->mt_conn_tls);
+
 		n = lws_ssl_client_connect1(wsi, (char *)wsi->a.context->pt[(int)wsi->tsi].serv_buf,
 					    wsi->a.context->pt_serv_buf_size);
 		lwsl_debug("%s: lws_ssl_client_connect1: %d\n", __func__, n);
@@ -194,8 +202,10 @@ lws_client_create_tls(struct lws *wsi, const char **pcce, int do_c1)
 			return CCTLS_RETURN_RETRY; /* caller should return 0 */
 		if (n < 0) {
 			*pcce = (const char *)wsi->a.context->pt[(int)wsi->tsi].serv_buf;
+			lws_metrics_caliper_report(wsi->cal_conn, METRES_NOGO);
 			return CCTLS_RETURN_ERROR;
 		}
+		/* ...connect1 already handled caliper if SSL_accept done */
 	} else
 		wsi->tls.ssl = NULL;
 

lib/tls/tls-network.c

@@ -204,10 +204,6 @@ lws_gate_accepts(struct lws_context *context, int on)
 
 	lwsl_notice("%s: on = %d\n", __func__, on);
 
-#if defined(LWS_WITH_STATS)
-	context->updated = 1;
-#endif
-
 	while (v) {
 		if (v->tls.use_ssl && v->lserv_wsi &&
 		    lws_change_pollfd(v->lserv_wsi, (LWS_POLLIN) * !on,

lib/tls/tls-server.c

@@ -157,9 +157,6 @@ lws_server_socket_service_ssl(struct lws *wsi, lws_sockfd_type accept_fd, char f
 			goto fail;
 		}
 
-#if defined(LWS_WITH_STATS)
-		context->updated = 1;
-#endif
 		/*
 		 * we are not accepted yet, but we need to enter ourselves
 		 * as a live connection.  That way we can retry when more
@@ -318,20 +315,13 @@ punt:
 
 		/* normal SSL connection processing path */
 
-#if defined(LWS_WITH_STATS)
-		/* only set this the first time around */
-		if (!wsi->accept_start_us)
-			wsi->accept_start_us = lws_now_usecs();
-#endif
 		errno = 0;
-		lws_stats_bump(pt, LWSSTATS_C_SSL_ACCEPT_SPIN, 1);
 		n = lws_tls_server_accept(wsi);
 		lwsl_info("SSL_accept says %d\n", n);
 		switch (n) {
 		case LWS_SSL_CAPABLE_DONE:
 			break;
 		case LWS_SSL_CAPABLE_ERROR:
-			lws_stats_bump(pt, LWSSTATS_C_SSL_CONNECTIONS_FAILED, 1);
 	                lwsl_info("%s: SSL_accept failed socket %u: %d\n",
 	                		__func__, wsi->desc.sockfd, n);
 			wsi->socket_is_permanently_unusable = 1;
@@ -341,26 +331,6 @@ punt:
 			return 0;
 		}
 
-		lws_stats_bump(pt, LWSSTATS_C_SSL_CONNECTIONS_ACCEPTED, 1);
-#if defined(LWS_WITH_STATS)
-		if (wsi->accept_start_us)
-			lws_stats_bump(pt,
-				      LWSSTATS_US_SSL_ACCEPT_LATENCY_AVG,
-				      lws_now_usecs() -
-					      wsi->accept_start_us);
-		wsi->accept_start_us = lws_now_usecs();
-#endif
-#if defined(LWS_WITH_DETAILED_LATENCY)
-		if (context->detailed_latency_cb) {
-			wsi->detlat.type = LDLT_TLS_NEG_SERVER;
-			wsi->detlat.latencies[LAT_DUR_PROXY_RX_TO_ONWARD_TX] =
-				(uint32_t)(lws_now_usecs() -
-				wsi->detlat.earliest_write_req_pre_write);
-			wsi->detlat.latencies[LAT_DUR_USERCB] = 0;
-			lws_det_lat_cb(wsi->a.context, &wsi->detlat);
-		}
-#endif
-
 		/* adapt our vhost to match the SNI SSL_CTX that was chosen */
 		vh = context->vhost_list;
 		while (vh) {

lwsws/main.c

@@ -64,7 +64,7 @@ static uv_signal_t signal_outer[2];
 static int pids[32];
 void lwsl_emit_stderr(int level, const char *line);
 
-#define LWSWS_CONFIG_STRING_SIZE (32 * 1024)
+#define LWSWS_CONFIG_STRING_SIZE (64 * 1024)
 
 static const struct lws_extension exts[] = {
 #if !defined(LWS_WITHOUT_EXTENSIONS)

minimal-examples/api-tests/api-test-lws_tokenize/main.c

@@ -204,7 +204,7 @@ struct tests tests[] = {
 	}, {
 		"fr-CH, fr;q=0.9, en;q=0.8, de;q=0.7, *;q=0.5",
 		expected7, LWS_ARRAY_SIZE(expected7),
-		LWS_TOKENIZE_F_RFC7230_DELIMS
+		LWS_TOKENIZE_F_ASTERISK_NONTERM | LWS_TOKENIZE_F_RFC7230_DELIMS
 	},
 	{
 		" Οὐχὶ ταὐτὰ παρίσταταί μοι γιγνώσκειν, ὦ ἄνδρες ᾿Αθηναῖοι, greek",
@@ -720,6 +720,10 @@ int main(int argc, const char **argv)
 		lwsl_user("%s: wc 15 fail\n", __func__);
 		fail++;
 	}
+	if (lws_strcmp_wildcard("ssproxy.n.cn.*", 14, "ssproxy.n.cn.failures")) {
+		lwsl_user("%s: wc 16 fail\n", __func__);
+		fail++;
+	}
 
 	lwsl_user("Completed: PASS: %d, FAIL: %d\n", ok, fail);
 

minimal-examples/http-client/minimal-http-client-multi/minimal-http-client-multi.c

@@ -217,6 +217,30 @@ static const struct lws_protocols protocols[] = {
 	{ NULL, NULL, 0, 0 }
 };
 
+#if defined(LWS_WITH_SYS_METRICS)
+
+static int
+my_metric_report(lws_metric_pub_t *mp)
+{
+	lws_metric_bucket_t *sub = mp->u.hist.head;
+	char buf[192];
+
+	do {
+		if (lws_metrics_format(mp, &sub, buf, sizeof(buf)))
+			lwsl_user("%s: %s\n", __func__, buf);
+	} while ((mp->flags & LWSMTFL_REPORT_HIST) && sub);
+
+	/* 0 = leave metric to accumulate, 1 = reset the metric */
+
+	return 1;
+}
+
+static const lws_system_ops_t system_ops = {
+	.metric_report = my_metric_report,
+};
+
+#endif
+
 static void
 signal_cb(void *handle, int signum)
 {
@@ -362,6 +386,11 @@ int main(int argc, const char **argv)
 	 * network wsi) that we will use.
 	 */
 	info.fd_limit_per_thread = 1 + COUNT + 1;
+	info.pcontext = &context;
+
+#if defined(LWS_WITH_SYS_METRICS)
+	info.system_ops = &system_ops;
+#endif
 
 #if defined(LWS_WITH_MBEDTLS) || defined(USE_WOLFSSL)
 	/*
@@ -374,11 +403,6 @@ int main(int argc, const char **argv)
 	if ((p = lws_cmdline_option(argc, argv, "--limit")))
 		info.simultaneous_ssl_restriction = atoi(p);
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	info.detailed_latency_cb = lws_det_lat_plot_cb;
-	info.detailed_latency_filepath = "/tmp/lws-latency-results";
-#endif
-
 	context = lws_create_context(&info);
 	if (!context) {
 		lwsl_err("lws init failed\n");

minimal-examples/http-server/minimal-http-server-tls/minimal-http-server-tls.c

@@ -23,8 +23,41 @@
 
 static int interrupted;
 
-static const struct lws_http_mount mount = {
+#if defined(LWS_WITH_PLUGINS)
+static const char * const plugin_dirs[] = {
+	LWS_INSTALL_DATADIR"/libwebsockets-test-server/plugins/",
+	NULL
+};
+#endif
+
+static const struct lws_http_mount
+#if defined(LWS_WITH_SYS_METRICS)
+	mount_metrics = {
 	/* .mount_next */		NULL,		/* linked-list "next" */
+	/* .mountpoint */		"/metrics",		/* mountpoint URL */
+	/* .origin */			"lws-openmetrics", /* serve from dir */
+	/* .def */			"x",	/* default filename */
+	/* .protocol */			"lws-openmetrics",
+	/* .cgienv */			NULL,
+	/* .extra_mimetypes */		NULL,
+	/* .interpret */		NULL,
+	/* .cgi_timeout */		0,
+	/* .cache_max_age */		0,
+	/* .auth_mask */		0,
+	/* .cache_reusable */		0,
+	/* .cache_revalidate */		0,
+	/* .cache_intermediaries */	0,
+	/* .origin_protocol */		LWSMPRO_CALLBACK, /* bind to callback */
+	/* .mountpoint_len */		8,		/* char count */
+	/* .basic_auth_login_file */	NULL,
+	},
+#endif
+	mount = {
+#if defined(LWS_WITH_SYS_METRICS)
+	/* .mount_next */		&mount_metrics,		/* linked-list "next" */
+#else
+	/* .mount_next */		NULL,		/* linked-list "next" */
+#endif
 	/* .mountpoint */		"/",		/* mountpoint URL */
 	/* .origin */			"./mount-origin", /* serve from dir */
 	/* .def */			"index.html",	/* default filename */
@@ -94,6 +127,10 @@ int main(int argc, const char **argv)
 	info.ssl_cert_filepath = "localhost-100y.cert";
 	info.ssl_private_key_filepath = "localhost-100y.key";
 
+#if defined(LWS_WITH_PLUGINS)
+	info.plugin_dirs = plugin_dirs;
+#endif
+
 	if (lws_cmdline_option(argc, argv, "-h"))
 		info.options |= LWS_SERVER_OPTION_VHOST_UPG_STRICT_HOST_CHECK;
 

minimal-examples/secure-streams/minimal-secure-streams-alexa/main.c

@@ -389,11 +389,6 @@ int main(int argc, const char **argv)
 	info.port = CONTEXT_PORT_NO_LISTEN;
 	info.pprotocols = protocols;
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	info.detailed_latency_cb = lws_det_lat_plot_cb;
-	info.detailed_latency_filepath = "/tmp/lws-latency-ssproxy";
-#endif
-
 	/* integrate us with lws system state management when context created */
 	nl.name = "app";
 	nl.notify_cb = app_system_state_nf;

minimal-examples/secure-streams/minimal-secure-streams-avs/main-client.c

@@ -100,11 +100,6 @@ int main(int argc, const char **argv)
 	info.protocols = lws_sspc_protocols;
 	info.port = CONTEXT_PORT_NO_LISTEN;
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	info.detailed_latency_cb = lws_det_lat_plot_cb;
-	info.detailed_latency_filepath = "/tmp/lws-latency-ssproxy";
-#endif
-
 	/* integrate us with lws system state management when context created */
 	nl.name = "app";
 	nl.notify_cb = app_system_state_nf;

minimal-examples/secure-streams/minimal-secure-streams-avs/main.c

@@ -341,11 +341,6 @@ int main(int argc, const char **argv)
 	}
 #endif
 
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	info.detailed_latency_cb = lws_det_lat_plot_cb;
-	info.detailed_latency_filepath = "/tmp/lws-latency-ssproxy";
-#endif
-
 	/* integrate us with lws system state management when context created */
 	nl.name = "app";
 	nl.notify_cb = app_system_state_nf;

minimal-examples/secure-streams/minimal-secure-streams-post/minimal-secure-streams-post.c

@@ -500,10 +500,6 @@ int main(int argc, const char **argv)
 	info.options = LWS_SERVER_OPTION_EXPLICIT_VHOSTS |
 		       LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT;
 #endif
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	info.detailed_latency_cb = lws_det_lat_plot_cb;
-	info.detailed_latency_filepath = "/tmp/lws-latency-ssproxy";
-#endif
 
 	/* integrate us with lws system state management when context created */
 

minimal-examples/secure-streams/minimal-secure-streams-proxy/main.c

@@ -206,6 +206,30 @@ static lws_state_notify_link_t * const app_notifier_list[] = {
 	&nl, NULL
 };
 
+#if defined(LWS_WITH_SYS_METRICS)
+
+static int
+my_metric_report(lws_metric_pub_t *mp)
+{
+	lws_metric_bucket_t *sub = mp->u.hist.head;
+	char buf[192];
+
+	do {
+		if (lws_metrics_format(mp, &sub, buf, sizeof(buf)))
+			lwsl_user("%s: %s\n", __func__, buf);
+	} while ((mp->flags & LWSMTFL_REPORT_HIST) && sub);
+
+	/* 0 = leave metric to accumulate, 1 = reset the metric */
+
+	return 1;
+}
+
+static const lws_system_ops_t system_ops = {
+	.metric_report = my_metric_report,
+};
+
+#endif
+
 static void
 sigint_handler(int sig)
 {
@@ -262,13 +286,9 @@ int main(int argc, const char **argv)
 	info.options = LWS_SERVER_OPTION_EXPLICIT_VHOSTS |
 		       LWS_SERVER_OPTION_H2_JUST_FIX_WINDOW_UPDATE_OVERFLOW |
 		       LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT;
-	info.fd_limit_per_thread = 1 + 32 + 1;
+	info.fd_limit_per_thread = 1 + 6 + 1;
 	info.pss_policies_json = default_ss_policy;
 	info.port = CONTEXT_PORT_NO_LISTEN;
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	info.detailed_latency_cb = lws_det_lat_plot_cb;
-	info.detailed_latency_filepath = "/tmp/lws-latency-ssproxy";
-#endif
 
 	/* integrate us with lws system state management when context created */
 	nl.name = "app";
@@ -278,6 +298,11 @@ int main(int argc, const char **argv)
 	info.pt_serv_buf_size = (unsigned int)((6144 * 2) + 2048);
 	info.max_http_header_data = (unsigned short)(6144 + 2048);
 
+#if defined(LWS_WITH_SYS_METRICS)
+	info.system_ops = &system_ops;
+	info.metrics_prefix = "ssproxy";
+#endif
+
 	context = lws_create_context(&info);
 	if (!context) {
 		lwsl_err("lws init failed\n");

minimal-examples/secure-streams/minimal-secure-streams-smd/minimal-secure-streams-smd.c

@@ -178,6 +178,7 @@ static const lws_ss_info_t ssi_lws_smd = {
 	.user_alloc		  = sizeof(myss_t),
 	.streamtype		  = LWS_SMD_STREAMTYPENAME,
 	.manual_initial_tx_credit = LWSSMDCL_SYSTEM_STATE |
+				    LWSSMDCL_METRICS |
 				    LWSSMDCL_NETWORK,
 };
 

minimal-examples/secure-streams/minimal-secure-streams-staticpolicy/minimal-secure-streams.c

@@ -230,10 +230,6 @@ int main(int argc, const char **argv)
 	info.options = LWS_SERVER_OPTION_EXPLICIT_VHOSTS |
 		       LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT;
 #endif
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	info.detailed_latency_cb = lws_det_lat_plot_cb;
-	info.detailed_latency_filepath = "/tmp/lws-latency-ssproxy";
-#endif
 
 	/* integrate us with lws system state management when context created */
 

minimal-examples/secure-streams/minimal-secure-streams-testsfail/minimal-secure-streams-testsfail.c

@@ -350,6 +350,58 @@ static const char * const default_ss_policy =
 			"\"opportunistic\": true,"
 			"\"retry\": \"default\","
 			"\"tls_trust_store\": \"arca1\""
+
+		"}},{"
+
+		/*
+		 * Various kinds of tls failure
+		 *
+		 * hostname.badcert.warmcat.com: serves valid cert but for
+		 *				 warmcat.com
+		 *
+		 * warmcat.com:446: serves valid but expired cert
+		 *
+		 * I don't have an easy way to make the test for "not valid yet"
+		 * cert without root
+		 *
+		 * invalidca.badcert.warmcat.com:  selfsigned cert for that
+		 *				   hostname
+		 */
+
+		    "\"badcert_hostname\": {"
+			"\"endpoint\": \"hostname.badcert.warmcat.com\","
+			"\"port\": 443,"
+			"\"protocol\": \"h1\","
+			"\"http_method\": \"GET\","
+			"\"http_url\": \"/\","
+			"\"tls\": true,"
+			"\"opportunistic\": true,"
+			"\"retry\": \"default\","
+			"\"tls_trust_store\": \"le_via_dst\""
+		"}},{"
+		    "\"badcert_expired\": {"
+			"\"endpoint\": \"warmcat.com\","
+			"\"port\": 446,"
+			"\"protocol\": \"h1\","
+			"\"http_method\": \"GET\","
+			"\"http_url\": \"/\","
+			"\"tls\": true,"
+			"\"opportunistic\": true,"
+			"\"retry\": \"default\","
+			"\"tls_trust_store\": \"le_via_dst\""
+		"}},{"
+		    "\"badcert_selfsigned\": {"
+			"\"endpoint\": \"invalidca.badcert.warmcat.com\","
+			"\"port\": 443,"
+			"\"protocol\": \"h1\","
+			"\"http_method\": \"GET\","
+			"\"http_url\": \"/\","
+			"\"tls\": true,"
+			"\"nghttp2_quirk_end_stream\": true,"
+			"\"h2q_oflow_txcr\": true,"
+			"\"opportunistic\": true,"
+			"\"retry\": \"default\","
+			"\"tls_trust_store\": \"le_via_dst\""
                 "}}"
 	"]}"
 ;
@@ -495,6 +547,29 @@ struct tests_seq {
 		12345
 	},
 
+	/*
+	 * Let's fail at the tls negotiation various ways
+	 */
+
+	{
+		"h1:badcert_hostname",
+		"badcert_hostname", 5 * LWS_US_PER_SEC, LWSSSCS_TIMEOUT,
+		(1 << LWSSSCS_QOS_NACK_REMOTE) |
+		(1 << LWSSSCS_ALL_RETRIES_FAILED)
+	},
+	{
+		"h1:badcert_expired",
+		"badcert_expired", 5 * LWS_US_PER_SEC, LWSSSCS_TIMEOUT,
+		(1 << LWSSSCS_QOS_NACK_REMOTE) |
+		(1 << LWSSSCS_ALL_RETRIES_FAILED)
+	},
+	{
+		"h1:badcert_selfsigned",
+		"badcert_selfsigned", 5 * LWS_US_PER_SEC, LWSSSCS_TIMEOUT,
+		(1 << LWSSSCS_QOS_NACK_REMOTE) |
+		(1 << LWSSSCS_ALL_RETRIES_FAILED)
+	},
+
 };
 
 typedef struct myss {
@@ -675,6 +750,29 @@ static lws_state_notify_link_t * const app_notifier_list[] = {
 	&nl, NULL
 };
 
+#if defined(LWS_WITH_SYS_METRICS)
+static int
+my_metric_report(lws_metric_pub_t *mp)
+{
+	lws_metric_bucket_t *sub = mp->u.hist.head;
+	char buf[192];
+
+	do {
+		if (lws_metrics_format(mp, &sub, buf, sizeof(buf)))
+			lwsl_user("%s: %s\n", __func__, buf);
+	} while ((mp->flags & LWSMTFL_REPORT_HIST) && sub);
+
+	/* 0 = leave metric to accumulate, 1 = reset the metric */
+
+	return 1;
+}
+
+static const lws_system_ops_t system_ops = {
+	.metric_report = my_metric_report,
+};
+
+#endif
+
 static void
 sigint_handler(int sig)
 {
@@ -740,6 +838,13 @@ main(int argc, const char **argv)
 	nl.notify_cb = app_system_state_nf;
 	info.register_notifier_list = app_notifier_list;
 
+#if defined(LWS_WITH_SYS_METRICS)
+	info.system_ops = &system_ops;
+#if defined(LWS_WITH_SECURE_STREAMS_PROXY_API)
+	info.metrics_prefix = "ssmex";
+#endif
+#endif
+
 	/* create the context */
 
 	context = lws_create_context(&info);

minimal-examples/secure-streams/minimal-secure-streams/minimal-secure-streams.c

@@ -125,7 +125,7 @@ static const char * const default_ss_policy =
 #if defined(VIA_LOCALHOST_SOCKS)
 			"\"http_url\":"		"\"policy/minimal-proxy-socks.json\","
 #else
-			"\"http_url\":"		"\"policy/minimal-proxy-2.json\","
+			"\"http_url\":"		"\"policy/minimal-proxy-v4.2.json\","
 #endif
 			"\"tls\":"		"true,"
 			"\"opportunistic\":"	"true,"
@@ -363,6 +363,30 @@ static lws_state_notify_link_t * const app_notifier_list[] = {
 	&nl, NULL
 };
 
+#if defined(LWS_WITH_SYS_METRICS)
+
+static int
+my_metric_report(lws_metric_pub_t *mp)
+{
+	lws_metric_bucket_t *sub = mp->u.hist.head;
+	char buf[192];
+
+	do {
+		if (lws_metrics_format(mp, &sub, buf, sizeof(buf)))
+			lwsl_user("%s: %s\n", __func__, buf);
+	} while ((mp->flags & LWSMTFL_REPORT_HIST) && sub);
+
+	/* 0 = leave metric to accumulate, 1 = reset the metric */
+
+	return 1;
+}
+
+static const lws_system_ops_t system_ops = {
+	.metric_report = my_metric_report,
+};
+
+#endif
+
 static void
 sigint_handler(int sig)
 {
@@ -425,10 +449,6 @@ int main(int argc, const char **argv)
 		       LWS_SERVER_OPTION_H2_JUST_FIX_WINDOW_UPDATE_OVERFLOW |
 		       LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT;
 #endif
-#if defined(LWS_WITH_DETAILED_LATENCY)
-	info.detailed_latency_cb = lws_det_lat_plot_cb;
-	info.detailed_latency_filepath = "/tmp/lws-latency-ssproxy";
-#endif
 
 	/* integrate us with lws system state management when context created */
 
@@ -436,6 +456,12 @@ int main(int argc, const char **argv)
 	nl.notify_cb = app_system_state_nf;
 	info.register_notifier_list = app_notifier_list;
 
+
+#if defined(LWS_WITH_SYS_METRICS)
+	info.system_ops = &system_ops;
+	info.metrics_prefix = "ssmex";
+#endif
+
 	/* create the context */
 
 	context = lws_create_context(&info);

plugins/CMakeLists.txt

@@ -147,13 +147,12 @@ if (LWS_ROLE_WS)
 			endif()
 		endif()
 
-	if (LWS_WITH_SERVER_STATUS)
+	if (LWS_WITH_SYS_METRICS)
-			create_plugin(protocol_lws_server_status ""
+			create_plugin(protocol_lws_openmetrics_export ""
-				      "protocol_lws_server_status.c" "" "")
+				      "protocol_lws_openmetrics_export.c" "" "")
 			if (NOT LWS_WITH_PLUGINS_BUILTIN)
-				target_compile_definitions(protocol_lws_server_status PRIVATE LWS_BUILDING_SHARED)
+				target_compile_definitions(protocol_lws_openmetrics_export PRIVATE LWS_BUILDING_SHARED)
 			endif()
-
 	endif()
 
 	if (NOT LWS_WITHOUT_CLIENT)
@@ -237,11 +236,6 @@ if (LWS_WITH_PLUGINS AND NOT LWS_WITH_PLUGINS_BUILTIN)
 	endif()
 
 
-if (LWS_WITH_SERVER_STATUS)
-	install(FILES server-status.html;server-status.js;server-status.css;lwsws-logo.png
-		DESTINATION share/libwebsockets-test-server/server-status
-			COMPONENT examples)
-endif()
 endif()
 
 export_to_parent_intermediate()

plugins/protocol_lws_server_status.c

@@ -1,218 +0,0 @@
-/*
- * libwebsockets-test-server - libwebsockets test implementation
- *
- * Written in 2010-2019 by Andy Green <andy@warmcat.com>
- *
- * This file is made available under the Creative Commons CC0 1.0
- * Universal Public Domain Dedication.
- *
- * The person who associated a work with this deed has dedicated
- * the work to the public domain by waiving all of his or her rights
- * to the work worldwide under copyright law, including all related
- * and neighboring rights, to the extent allowed by law. You can copy,
- * modify, distribute and perform the work, even for commercial purposes,
- * all without asking permission.
- *
- * The test apps are intended to be adapted for use in your code, which
- * may be proprietary.  So unlike the library itself, they are licensed
- * Public Domain.
- */
-
-#define LWS_DLL
-#define LWS_INTERNAL
-#include <libwebsockets.h>
-#include <string.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <stdio.h>
-
-struct lws_ss_filepath {
-	struct lws_ss_filepath *next;
-	char filepath[128];
-};
-
-struct lws_ss_dumps {
-	char buf[32768];
-	int length;
-};
-
-struct pss {
-	int ver;
-	int pos;
-};
-
-struct vhd {
-	struct lws_context *context;
-	struct lws_vhost *vhost;
-	const struct lws_protocols *protocol;
-	lws_sorted_usec_list_t sul;
-	int hide_vhosts;
-	int tow_flag;
-	int period_s;
-	int clients;
-	struct lws_ss_dumps d;
-	struct lws_ss_filepath *fp;
-};
-
-static const struct lws_protocols protocols[1];
-
-static void
-update(struct lws_sorted_usec_list *sul)
-{
-	struct vhd *v = lws_container_of(sul, struct vhd, sul);
-	struct lws_ss_filepath *fp;
-	char contents[256], pure[256], *p = v->d.buf + LWS_PRE,
-	     *end = v->d.buf + sizeof(v->d.buf) - LWS_PRE - 1;
-	int n, first = 1, fd;
-
-	p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), "{\"i\":");
-	p += lws_json_dump_context(v->context, p, lws_ptr_diff(end, p),
-				   v->hide_vhosts);
-	p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), ", \"files\": [");
-
-	fp = v->fp;
-	while (fp) {
-		if (!first)
-			p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), ",");
-
-		strcpy(pure, "(unknown)");
-		fd = lws_open(fp->filepath, LWS_O_RDONLY);
-		if (fd >= 0) {
-			n = (int)read(fd, contents, sizeof(contents) - 1);
-			close(fd);
-			if (n >= 0) {
-				contents[n] = '\0';
-				lws_json_purify(pure, contents, sizeof(pure), NULL);
-			}
-		}
-
-		p += lws_snprintf(p, lws_ptr_diff_size_t(end, p),
-				"{\"path\":\"%s\",\"val\":\"%s\"}",
-					fp->filepath, pure);
-		first = 0;
-
-		fp = fp->next;
-	}
-	p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), "]}");
-	v->d.length = lws_ptr_diff(p, (v->d.buf + LWS_PRE));
-
-	lws_callback_on_writable_all_protocol(v->context, &protocols[0]);
-
-	lws_sul_schedule(v->context, 0, &v->sul, update, v->period_s * LWS_US_PER_SEC);
-}
-
-static int
-callback_lws_server_status(struct lws *wsi, enum lws_callback_reasons reason,
-			   void *user, void *in, size_t len)
-{
-	const struct lws_protocol_vhost_options *pvo =
-			(const struct lws_protocol_vhost_options *)in;
-	struct vhd *v = (struct vhd *)
-			lws_protocol_vh_priv_get(lws_get_vhost(wsi),
-					lws_get_protocol(wsi));
-	struct lws_ss_filepath *fp, *fp1, **fp_old;
-	int m;
-
-	switch (reason) {
-
-	case LWS_CALLBACK_ESTABLISHED:
-		lwsl_info("%s: LWS_CALLBACK_ESTABLISHED\n", __func__);
-		if (!v->clients++) {
-			lws_sul_schedule(lws_get_context(wsi), 0, &v->sul, update, 1);
-			lwsl_info("%s: starting updates\n", __func__);
-		}
-		break;
-
-	case LWS_CALLBACK_CLOSED:
-		if (!--v->clients)
-			lwsl_notice("%s: stopping updates\n", __func__);
-
-		break;
-
-	case LWS_CALLBACK_PROTOCOL_INIT: /* per vhost */
-		if (v)
-			break;
-
-		lws_protocol_vh_priv_zalloc(lws_get_vhost(wsi),
-					    lws_get_protocol(wsi),
-					    sizeof(struct vhd));
-		v = (struct vhd *)lws_protocol_vh_priv_get(lws_get_vhost(wsi),
-							   lws_get_protocol(wsi));
-
-		fp_old = &v->fp;
-
-		while (pvo) {
-			if (!strcmp(pvo->name, "hide-vhosts"))
-				v->hide_vhosts = atoi(pvo->value);
-			if (!strcmp(pvo->name, "update-ms"))
-				v->period_s = (atoi(pvo->value) + 500) / 1000;
-			else
-				v->period_s = 5;
-			if (!strcmp(pvo->name, "filepath")) {
-				fp = malloc(sizeof(*fp));
-				if (!fp)
-					return -1;
-				fp->next = NULL;
-				lws_snprintf(&fp->filepath[0],
-					     sizeof(fp->filepath), "%s",
-					     pvo->value);
-				*fp_old = fp;
-				fp_old = &fp->next;
-			}
-			pvo = pvo->next;
-		}
-		v->context = lws_get_context(wsi);
-		v->vhost = lws_get_vhost(wsi);
-		v->protocol = lws_get_protocol(wsi);
-
-		lws_sul_schedule(lws_get_context(wsi), 0, &v->sul, update, 1);
-		break;
-
-	case LWS_CALLBACK_PROTOCOL_DESTROY: /* per vhost */
-		if (!v)
-			break;
-		fp = v->fp;
-		while (fp) {
-			fp1= fp->next;
-			free(fp);
-			fp = fp1;
-		}
-		break;
-
-	case LWS_CALLBACK_SERVER_WRITEABLE:
-		m = lws_write(wsi, (unsigned char *)v->d.buf + LWS_PRE,
-			      (size_t)v->d.length, LWS_WRITE_TEXT);
-		if (m < 0)
-			return -1;
-		break;
-
-	default:
-		break;
-	}
-
-	return 0;
-}
-
-static const struct lws_protocols protocols[] = {
-	{
-		"lws-server-status",
-		callback_lws_server_status,
-		sizeof(struct pss),
-		1024,
-	},
-};
-
-LWS_VISIBLE const lws_plugin_protocol_t lws_server_status = {
-	.hdr = {
-		"lws server status",
-		"lws_protocol_plugin",
-		LWS_PLUGIN_API_MAGIC
-	},
-
-	.protocols = protocols,
-	.count_protocols = LWS_ARRAY_SIZE(protocols),
-	.extensions = NULL,
-	.count_extensions = 0,
-};

plugins/server-status.html

@@ -1,25 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
- <meta charset=utf-8 http-equiv="Content-Language" content="en"/>
- <link rel="stylesheet" type="text/css" href="server-status.css"/>
- <script src="/lws-common.js"></script>
- <script type='text/javascript' src='server-status.js'></script>
- <title>LWS Server Status</title>
-</head>
-
-<body>
-<header></header>
-<article>
-
-<table>
-<tr><td><img src="./lwsws-logo.png"></td><td>
-<span id=title class=title>Server status</span></td></tr>
-<tr><td align=center colspan=2>
-<div id="conninfo">...</div>
-<div id="json"></div>
-</td></tr>
-</table>
-</article>
-</body>
-</html>