Mirrors/libwebsockets
1
0
Fork 0
mirror of https://github.com/warmcat/libwebsockets.git synced 2025-03-09 00:00:04 +01:00
Code Issues Releases Wiki Activity

lecp: add CBOR stream parser LECP like JSON LEJP

Browse source 
This provides very memory-efficient CBOR stream parsing
and writing.

The parser  converts pieces of CBOR into callbacks that define
the structure and collate string and blobs into buffer chunks
for extensible and easy access.

It is fragementation-safe and does not need all the CBOR in
the same place at one time, chunks of CBOR are parsed and
discarded as provided.

It does not allocate and just needs a few hundred bytes of
stack for even huge CBOR objects.  Huge strings and blobs
are handled without needing memory to hold them atomically.

Includes ./minimal-examples/api-tests/api-test-lecp that
unit tests it against 82 official example CBORs and
26 additional test vectors from COSE (just checking the CBOR
parsing).

The writing apis allow printf style semantics with a variety
of CBOR-aware %-formats.  The apis write into a context that
manages output buffer usage, if the output buffer fills,
then the apis return with an AGAIN code that lets you issue
and reset the output buffer and repeat the api all to issue
more output.  The subsequent calls can occur much later or
from a different function context, so this is perfect for
WRITEABLE-mediated output from the network parts of lws.

See ./READMEs/README.cbor-lecp.md
This commit is contained in:
Andy Green 2021-06-30 04:58:25 +01:00
parent b31c5d6ffe
commit dcaa0013b4
18 changed files with 8245 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.sai.json
View file

@ -130,7 +130,7 @@
"platforms": "w10/x86_64-amd/msvc, w10/x86_64-amd/noptmsvc, freertos-linkit/arm32-m4-mt7697-usi/gcc, linux-ubuntu-2004/aarch64-a72-bcm2711-rpi4/gcc, w10/x86_64-amd/mingw32, w10/x86_64-amd/mingw64, netbsd/aarch64BE-bcm2837-a53/gcc, netbsd/x86_64-amd/gcc, w10/x86_64-amd/wmbedtlsmsvc"
},
"fault-injection": {
"cmake": "-DLWS_WITH_SYS_FAULT_INJECTION=1 -DLWS_WITH_MINIMAL_EXAMPLES=1",
"cmake": "-DLWS_WITH_SYS_FAULT_INJECTION=1 -DLWS_WITH_MINIMAL_EXAMPLES=1 -DLWS_WITH_CBOR=1",
"platforms": "w10/x86_64-amd/msvc"
},
"esp32-heltec": {
@ -139,7 +139,7 @@
"platforms": "none, freertos-espidf/xl6-esp32/gcc"
},
"esp32-wrover": {
"cmake": "-DLWS_IPV6=0",
"cmake": "-DLWS_IPV6=0 -DLWS_WITH_CBOR=1",
"cpack": "esp-wrover-kit",
"platforms": "none, freertos-espidf/xl6-esp32/gcc"
},

2
CMakeLists.txt
View file

@ -231,6 +231,8 @@ option(LWS_WITHOUT_BUILTIN_SHA1 "Don't build the lws sha-1 (eg, because openssl
option(LWS_WITHOUT_DAEMONIZE "Don't build the daemonization api" ON)
option(LWS_SSL_SERVER_WITH_ECDH_CERT "Include SSL server use ECDH certificate" OFF)
option(LWS_WITH_LEJP "With the Lightweight JSON Parser" ON)
option(LWS_WITH_CBOR "With the Lightweight LECP CBOR Parser" OFF)
option(LWS_WITH_CBOR_FLOAT "Build floating point types if building CBOR LECP" ON)
option(LWS_WITH_SQLITE3 "Require SQLITE3 support" OFF)
option(LWS_WITH_STRUCT_JSON "Generic struct serialization to and from JSON" OFF)
option(LWS_WITH_STRUCT_SQLITE3 "Generic struct serialization to and from SQLITE3" OFF)

28
LICENSE
View file

@ -8,9 +8,10 @@ Original liberal license retained:
- lib/misc/sha-1.c - 3-clause BSD license retained, link to original [BSD3]
- win32port/zlib - ZLIB license (see zlib.h) [ZLIB]
- lib/tls/mbedtls/wrapper - Apache 2.0 (only built if linked against mbedtls) {APACHE2]
- lib/tls/mbedtls/wrapper - Apache 2.0 (only built if linked against mbedtls) [APACHE2]
lib/tls/mbedtls/mbedtls-extensions.c
- lib/misc/base64-decode.c - already MIT
- lib/misc/ieeehalfprecision.c - 2-clause BSD license retained [BSD2]
Relicensed to MIT:
@ -55,6 +56,31 @@ https://opensource.org/licenses/MIT
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
IN THE SOFTWARE.
## BSD2
```
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the distribution
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
```
## BSD3

344
READMEs/README.cbor-lecp.md Normal file
View file

@ -0,0 +1,344 @@
# RFC8949 CBOR Stream Parsing and Writing
|||
|---|---|---|
|cmake| `LWS_WITH_CBOR`, `LWS_WITH_CBOR_FLOAT`|
|Header| ./include/libwebsockets/lws-lecp.h|
|api-test| ./minimal-examples/api-tests/api-test-lecp/|
|test app| ./test-apps/test-lecp.c -> libwebsockets-test-lecp|
LECP is the RFC8949 CBOR stream parsing counterpart to LEJP for JSON.
## Features
- Completely immune to input fragmentation, give it any size blocks of CBOR as
they become available; 1 byte, or 100K at a time give identical parsing
results
- Input chunks discarded as they are parsed, whole CBOR never needed in memory
- Nonrecursive, fixed stack usage of a few dozen bytes
- No heap allocations at all, just requires ~500 byte context usually on
caller stack
- Creates callbacks to a user-provided handler as members are parsed out
- No payload size limit, supports huge / endless strings or blobs bigger than
system memory
- Collates utf-8 text and blob payloads into a 250-byte chunk buffer for ease
of access
- Write apis don't use any heap allocations or recursion either
- Write apis use an explicit context with its own lifecycle, and printf style
vaargs including sized blobs, C strings, double, int, unsigned long etc
- Completely immune to output fragmentation, supports huge strings and blobs
into small buffers, api returns to indicates unfinished if it needs to be
called again to continue; 1 byte or 100K output buffer give same results
- Write apis completely fill available buffer and if unfinished, continues
into same or different buffer when called again with same args; no
requirement for subsequent calls to be done sequentially or even from same
function
## Type limits
CBOR allows negative integers of up to 64 bits, these do not fit into a `uint64_t`.
LECP has a union for numbers that includes the types `uint64_t` and `int64_t`,
but it does not separately handle negative integers. Only -2^63.. 2^64 -1 can
be handled by the C types, the oversize negative numbers wrap and should be
avoided.
## Floating point support
Floats are handled using the IEEE memory format, it means they can be parsed
from the CBOR without needing any floating point support in the build. If
floating point is available, you can also enable `LWS_WITH_CBOR_FLOAT` and
a `float` and `double` types are available in the number item union. Otherwise
these are handled as `ctx->item.u.u32` and `ctx->item.u.u64` union members.
Half-float (16-bit) is defined in CBOR and always handled as a `uint16_t`
number union member `ctx->item.u.hf`.
## Callback reasons
The user callback does not have to handle any callbacks, it only needs to
process the data for the ones it is interested in.
|Callback reason|CBOR structure|Associated data|
|---|---|---|
|`LECPCB_CONSTRUCTED`|Created the parse context||
|`LECPCB_DESTRUCTED`|Destroyed the parse context||
|`LECPCB_COMPLETE`|The parsing completed OK||
|`LECPCB_FAILED`|The parsing failed||
|`LECPCB_VAL_TRUE`|boolean true||
|`LECPCB_VAL_FALSE`|boolean false||
|`LECPCB_VAL_NULL`|explicit NULL||
|`LECPCB_VAL_NUM_INT`|signed integer|`ctx->item.u.i64`|
|`LECPCB_VAL_STR_START`|A UTF-8 string is starting||
|`LECPCB_VAL_STR_CHUNK`|The next string chunk|`ctx->npos` bytes in `ctx->buf`|
|`LECPCB_VAL_STR_END`|The last string chunk|`ctx->npos` bytes in `ctx->buf`|
|`LECPCB_ARRAY_START`|An array is starting||
|`LECPCB_ARRAY_END`|An array has ended||
|`LECPCB_OBJECT_START`|A CBOR map is starting||
|`LECPCB_OBJECT_END`|A CBOR map has ended||
|`LECPCB_TAG_START`|The following data has a tag index|`ctx->item.u.u64`|
|`LECPCB_TAG_END`|The end of the data referenced by the last tag||
|`LECPCB_VAL_NUM_UINT`|Unsigned integer|`ctx->item.u.u64`|
|`LECPCB_VAL_UNDEFINED`|CBOR undefined||
|`LECPCB_VAL_FLOAT16`|half-float available as host-endian `uint16_t`|`ctx->item.u.hf`|
|`LECPCB_VAL_FLOAT32`|`float` (`uint32_t` if no float support) available|`ctx->item.u.f`|
|`LECPCB_VAL_FLOAT64`|`double` (`uint64_t` if no float support) available|`ctx->item.u.d`|
|`LECPCB_VAL_SIMPLE`|CBOR simple|`ctx->item.u.u64`|
|`LECPCB_VAL_BLOB_START`|A binary blob is starting||
|`LECPCB_VAL_BLOB_CHUNK`|The next blob chunk|`ctx->npos` bytes in `ctx->buf`|
|`LECPCB_VAL_BLOB_END`|The last blob chunk|`ctx->npos` bytes in `ctx->buf`|
|`LECPCB_ARRAY_ITEM_START`|A logical item in an array is starting|
|`LCEPDB_ARRAY_ITEM_END`|A logical item in an array has completed|
## CBOR indeterminite lengths
Indeterminite lengths are supported, but are concealed in the parser as far as
possible, the CBOR lengths or its indeterminacy are not exposed in the callback
interface at all, just chunks of data that may be the start, the middle, or the
end.
## Handling CBOR UTF-8 strings and blobs
When a string or blob is parsed, an advisory callback of `LECPCB_VAL_STR_START` or
`LECPCB_VAL_BLOB_START` occurs first. The `_STR_` callbacks indicate the
content is a CBOR UTF-8 string, `_BLOB_` indicates it is binary data.
Strings or blobs may have indeterminite length, but if so, they are composed
of logical chunks which must have known lengths. When the `_START` callback
occurs, the logical length either of the whole string, or of the sub-chunk if
indeterminite length, can be found in `ctx->item.u.u64`.
Payload is collated into `ctx->buf[]`, the valid length is in `ctx->npos`.
For short strings or blobs where the length is known, the whole payload is
delivered in a single `LECPCB_VAL_STR_END` or `LECPCB_VAL_BLOB_END` callback.
For payloads larger than the size of `ctx->buf[]`, `LECPCB_VAL_STR_CHUNK` or
`LECPCB_VAL_BLOB_CHUNK` callbacks occur delivering each sequential bufferload.
If the CBOR indicates the total length, the last chunk is delievered in a
`LECPCB_VAL_STR_END` or `LECPCB_VAL_BLOB_END`.
If the CBOR indicates the string end after the chunk, a zero-length `..._END`
callback is provided.
## Handling CBOR tags
CBOR tags are exposed as `LECPCB_TAG_START` and `LECPCB_TAG_END` pairs, at
the `_START` callback the tag index is available in `ctx->item.u.u64`.
## CBOR maps
You can check if you are on the "key" part of a map "key:value" pair using the
helper api `lecp_parse_map_is_key(ctx)`.
## Parsing paths
LECP maintains a "parsing path" in `ctx->path` that represents the context of
the callback events. As a convenience, at LECP context creation time, you can
pass in an array of path strings you want to match on, and have any match
checkable in the callback using `ctx->path_match`, it's 0 if no active match,
or the match index from your path array starting from 1 for the first entry.
|CBOR element|Representation in path|
|---|---|
|CBOR Array|`[]`|
|CBOR Map|`.`|
|CBOR Map entry key string|`keystring`|
## Accessing raw CBOR subtrees
Some CBOR usages like COSE require access to selected raw CBOR from the input
stream. `lecp_parse_report_raw(ctx, on)` lets you turn on and off buffering of
raw CBOR and reporting it in the parse callback with `LECPCB_LITERAL_CBOR`
callbacks. The callbacks mean the temp buffer `ctx->cbor[]` has `ctx->cbor_pos`
bytes of raw CBOR available in it. Callbacks are triggered when the buffer
fills, or reporting is turned off and the buffer has something in it.
By turning the reporting on and off according to the outer CBOR parsing state,
it's possible to get exactly the raw CBOR subtree that's needed.
Capturing and reporting the raw CBOR does not change that the same CBOR is being
passed to the parser as usual as well.
## Comparison with LEJP (JSON parser)
LECP is based on the same principles as LEJP and shares most of the callbacks.
The major differences:
- LEJP value callbacks all appear in `ctx->buf[]`, ie, floating-point is
provided to the callback in ascii form like `"1.0"`. CBOR provides a more
strict typing system, and the different type values are provided either in
`ctx->buf[]` for blobs or utf-8 text strtings, or the `item.u` union for
converted types, with additional callback reasons specific to each type.
- CBOR "maps" use `_OBJECT_START` and `_END` parsing callbacks around the
key / value pairs. LEJP has a special callback type `PAIR_NAME` for the
key string / integer, but in LECP these are provided as generic callbacks
dependent on type, ie, generic string callbacks or integer ones, and the
value part is represented according to whatever comes.
# Writing CBOR
CBOR is written into a `lws_lec_pctx_t` object that has been initialized to
point to an output buffer of a specified size, using printf type formatting.
Output is paused if the buffer fills, and the write api may be called again
later with the same context object, to resume emitting to the same or different
buffer.
This allows bufferloads of encoded CBOR to be produced on demand, it's designed
to fit usage in WRITEABLE callbacks and Secure Streams tx() callbacks where the
buffer size for one packet is already fixed.
CBOR array and map lengths are deduced from the format string, as is whether to
use indeterminite length formatting or not. For indeterminite text or binary
strings, a container of < >
|Format|Arg(s)|Meaning|
|---|---|---|
|`123`||unsigned literal number|
|`-123`||signed literal number|
|`%u`|`unsigned int`|number|
|`%lu`|`unsigned long int`|number|
|`%llu`|`unsigned long long int`|number|
|`%d`|`signed int`|number|
|`%ld`|`signed long int`|number|
|`%lld`|`signed long long int`|number|
|`%f`|`double`|floating point number|
|`123(...)`||literal tag and scope|
|`%t(...)`|`unsigned int`|tag and scope|
|`%lt(...)`|`unsigned long int`|tag and scope|
|`%llt(...)`|`unsigned long long int`|tag and scope|
|`[...]`||Array (fixed len if `]` in same format string)|
|`{...}`||Map (fixed len if `}` in same format string)|
|`<t...>`||Container for indeterminite text string frags|
|`<b...>`||Container for indeterminite binary string frags|
|`'string'`||Literal text of known length|
|`%s`|`const char *`|NUL-terminated string|
|`%.*s`|`int`, `const char *`|length-specified string|
|`%.*b`|`int`, `const uint8_t *`|length-specified binary|
|`:`||separator between Map items (a:b)|
|`,`||separator between Map pairs or array items|
Backslash is used as an escape in `'...'` literal strings, so `'\\'` represents
a string consisting of a single backslash, and `'\''` a string consisting of a
single single-quote.
For integers, various natural C types are available, but in all cases, the
number is represented in CBOR using the smallest valid way based on its value,
the long or long-long modifiers just apply to the expected C type in the args.
For floats, the C argument is always expected to be a `double` type following
C type promotion, but again it is represented in CBOR using the smallest valid
way based on value, half-floats are used for NaN / Infinity and where possible
for values like 0.0 and -1.0.
## Examples
### Literal ints
```
uint8_t buf[128];
lws_lec_pctx_t cbw;
lws_lec_init(&cbw, buf, sizeof(buf));
lws_lec_printf(ctx, "-1");
```
|||
|---|---|
|Return| `LWS_LECPCTX_RET_FINISHED`|
|`ctx->used`|1|
|`buf[]`|20|
### Dynamic ints
```
uint8_t buf[128];
lws_lec_pctx_t cbw;
int n = -1; /* could be long */
lws_lec_init(&cbw, buf, sizeof(buf));
lws_lec_printf(ctx, "%d", n); /* use %ld for long */
```
|||
|---|---|
|Return| `LWS_LECPCTX_RET_FINISHED`|
|`ctx->used`|1|
|`buf[]`|20|
### Maps, arrays and dynamic ints
```
...
int args[3] = { 1, 2, 3 };
lws_lec_printf(ctx, "{'a':%d,'b':[%d,%d]}", args[0], args[1], args[2]);
```
|||
|---|---|
|Return| `LWS_LECPCTX_RET_FINISHED`|
|`ctx->used`|9|
|`buf[]`|A2 61 61 01 61 62 82 02 03|
### String longer than the buffer
Using `%s` and the same string as an arg gives same results
```
uint8_t buf[16];
lws_lec_pctx_t cbw;
lws_lec_init(&cbw, buf, sizeof(buf));
lws_lec_printf(ctx, "'A literal string > one buf'");
/* not required to be in same function context or same buf,
* but the string must remain the same */
lws_lec_setbuf(&cbw, buf, sizeof(buf));
lws_lec_printf(ctx, "'A literal string > one buf'");
```
First call
|||
|---|---|
|Return| `LWS_LECPCTX_RET_AGAIN`|
|`ctx->used`|16|
|`buf[]`|78 1A 41 20 6C 69 74 65 72 61 6C 20 73 74 72 69|
Second call
|||
|---|---|
|Return| `LWS_LECPCTX_RET_FINISHED`|
|`ctx->used`|12|
|`buf[]`|6E 67 20 3E 20 6F 6E 65 20 62 75 66|
### Binary blob longer than the buffer
```
uint8_t buf[16], blob[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18 };
lws_lec_pctx_t cbw;
lws_lec_init(&cbw, buf, sizeof(buf));
lws_lec_printf(ctx, "%.*b", (int)sizeof(blob), blob);
/* not required to be in same function context or same buf,
* but the length and blob must remain the same */
lws_lec_setbuf(&cbw, buf, sizeof(buf));
lws_lec_printf(ctx, "%.*b", (int)sizeof(blob), blob);
```
First call
|||
|---|---|
|Return| `LWS_LECPCTX_RET_AGAIN`|
|`ctx->used`|16|
|`buf[]`|52 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F|
Second call
|||
|---|---|
|Return| `LWS_LECPCTX_RET_FINISHED`|
|`ctx->used`|3|
|`buf[]`|10 11 12|

107
READMEs/README.json-lejp.md Normal file
View file

@ -0,0 +1,107 @@
# LEJP JSON Stream Parser
|||
|---|---|---|
|cmake| `LWS_WITH_LEJP`|
|Header| ./include/libwebsockets/lws-lejp.h|
|api-test| ./minimal-examples/api-tests/api-test-lejp/|
|test app| ./test-apps/test-lejp.c -> libwebsockets-test-lejp|
LEJP is a lightweight JSON stream parser.
The features are:
- completely immune to input fragmentation, give it any size blocks of JSON as
they become available, 1 byte, or 100K at a time give identical parsing
results
- input chunks discarded as they are parsed, whole JSON never needed in memory
- nonrecursive, fixed stack usage of a few dozen bytes
- no heap allocations at all, just requires ~500 byte context usually on
caller stack
- creates callbacks to a user-provided handler as members are parsed out
- no payload size limit, supports huge / endless strings bigger than
system memory
- collates utf-8 text payloads into a 250-byte chunk buffer in the json parser
context object for ease of access
## Type handling
LEJP leaves all numbers in text form, they are signalled in different callbacks
according to int or float, but delivered as text strings in the first
`ctx->npos` chars of `ctx->buf`.
For numeric types, you would typically use `atoi()` or similar to recover the
number as a host type.
## Callback reasons
The user callback does not have to handle any callbacks, it only needs to
process the data for the ones it is interested in.
|Callback reason|JSON structure|Associated data|
|---|---|---|
|`LEJPCB_CONSTRUCTED`|Created the parse context||
|`LEJPCB_DESTRUCTED`|Destroyed the parse context||
|`LEJPCB_COMPLETE`|The parsing completed OK||
|`LEJPCB_FAILED`|The parsing failed||
|`LEJPCB_VAL_TRUE`|boolean true||
|`LEJPCB_VAL_FALSE`|boolean false||
|`LEJPCB_VAL_NULL`|explicit NULL||
|`LEJPCB_PAIR_NAME`|The name part of a JSON `key: value` map pair|`ctx->buf`|
|`LEJPCB_VAL_STR_START`|A UTF-8 string is starting||
|`LEJPCB_VAL_STR_CHUNK`|The next string chunk|`ctx->npos` bytes in `ctx->buf`|
|`LEJPCB_VAL_STR_END`|The last string chunk|`ctx->npos` bytes in `ctx->buf`|
|`LEJPCB_ARRAY_START`|An array is starting||
|`LEJPCB_ARRAY_END`|An array has ended||
|`LEJPCB_OBJECT_START`|A JSON object is starting||
|`LEJPCB_OBJECT_END`|A JSON object has ended||
## Handling JSON UTF-8 strings
When a string is parsed, an advisory callback of `LECPCB_VAL_STR_START` occurs
first. No payload is delivered with the START callback.
Payload is collated into `ctx->buf[]`, the valid length is in `ctx->npos`.
For short strings or blobs where the length is known, the whole payload is
delivered in a single `LECPCB_VAL_STR_END` callback.
For payloads larger than the size of `ctx->buf[]`, `LECPCB_VAL_STR_CHUNK`
callbacks occur delivering each sequential bufferload.
The last chunk (which may be zero length) is delievered by `LECPCB_VAL_STR_END`.
## Parsing paths
LEJP maintains a "parsing path" in `ctx->path` that represents the context of
the callback events. As a convenience, at LEJP context creation time, you can
pass in an array of path strings you want to match on, and have any match
checkable in the callback using `ctx->path_match`, it's 0 if no active match,
or the match index from your path array starting from 1 for the first entry.
|CBOR element|Representation in path|
|---|---|
|JSON Array|`[]`|
|JSON Map|`.`|
|JSON Map entry key string|`keystring`|
## Comparison with LECP (CBOR parser)
LECP is based on the same principles as LEJP and shares most of the callbacks.
The major differences:
- LEJP value callbacks all appear in `ctx->buf[]`, ie, floating-point is
provided to the callback in ascii form like `"1.0"`. CBOR provides a more
strict typing system, and the different type values are provided either in
`ctx->buf[]` for blobs or utf-8 text strtings, or the `item.u` union for
converted types, with additional callback reasons specific to each type.
- CBOR "maps" use `_OBJECT_START` and `_END` parsing callbacks around the
key / value pairs. LEJP has a special callback type `PAIR_NAME` for the
key string / integer, but in LECP these are provided as generic callbacks
dependent on type, ie, generic string callbacks or integer ones, and the
value part is represented according to whatever comes.

2
cmake/lws_config.h.in
View file

@ -167,6 +167,8 @@
#cmakedefine LWS_WITH_HTTP_UNCOMMON_HEADERS
#cmakedefine LWS_WITH_IPV6
#cmakedefine LWS_WITH_JOSE
#cmakedefine LWS_WITH_CBOR
#cmakedefine LWS_WITH_CBOR_FLOAT
#cmakedefine LWS_WITH_LEJP
#cmakedefine LWS_WITH_LIBEV
#cmakedefine LWS_WITH_LIBEVENT

1
include/libwebsockets.h
View file

@ -617,6 +617,7 @@ struct lws;
#include <libwebsockets/lws-vfs.h>
#endif
#include <libwebsockets/lws-lejp.h>
#include <libwebsockets/lws-lecp.h>
#include <libwebsockets/lws-struct.h>
#include <libwebsockets/lws-threadpool.h>
#include <libwebsockets/lws-tokenize.h>

542
include/libwebsockets/lws-lecp.h Normal file
View file

@ -0,0 +1,542 @@
/*
* libwebsockets - small server side websockets and web server implementation
*
* Copyright (C) 2010 - 2021 Andy Green <andy@warmcat.com>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
/** \defgroup lecp CBOR parser
* ##CBOR parsing related functions
* \ingroup lwsapi
*
* LECP is an extremely lightweight CBOR stream parser included in lws. It
* is aligned in approach with the LEJP JSON stream parser, with some additional
* things needed for CBOR.
*/
//@{
#ifndef LECP_MAX_PARSING_STACK_DEPTH
#define LECP_MAX_PARSING_STACK_DEPTH 5
#endif
#ifndef LECP_MAX_DEPTH
#define LECP_MAX_DEPTH 12
#endif
#ifndef LECP_MAX_INDEX_DEPTH
#define LECP_MAX_INDEX_DEPTH 8
#endif
#ifndef LECP_MAX_PATH
#define LECP_MAX_PATH 128
#endif
#ifndef LECP_STRING_CHUNK
/* must be >= 30 to assemble floats */
#define LECP_STRING_CHUNK 254
#endif
#define LECP_FLAG_CB_IS_VALUE 64
/*
* CBOR initial byte 3 x MSB bits are these
*/
enum {
LWS_CBOR_MAJTYP_UINT = 0 << 5,
LWS_CBOR_MAJTYP_INT_NEG = 1 << 5,
LWS_CBOR_MAJTYP_BSTR = 2 << 5,
LWS_CBOR_MAJTYP_TSTR = 3 << 5,
LWS_CBOR_MAJTYP_ARRAY = 4 << 5,
LWS_CBOR_MAJTYP_MAP = 5 << 5,
LWS_CBOR_MAJTYP_TAG = 6 << 5,
LWS_CBOR_MAJTYP_FLOAT = 7 << 5, /* also BREAK */
LWS_CBOR_MAJTYP_MASK = 7 << 5,
/*
* For the low 5 bits of the opcode, 0-23 are literals, unless it's
* FLOAT.
*
* 24 = 1 byte; 25 = 2..., 26 = 4... and 27 = 8 bytes following literal.
*/
LWS_CBOR_1 = 24,
LWS_CBOR_2 = 25,
LWS_CBOR_4 = 26,
LWS_CBOR_8 = 27,
LWS_CBOR_RESERVED = 28,
LWS_CBOR_SUBMASK = 0x1f,
/*
* Major type 7 discriminators in low 5 bits
* 0 - 23 is SIMPLE implicit value (like, eg, LWS_CBOR_SWK_TRUE)
*/
LWS_CBOR_SWK_FALSE = 20,
LWS_CBOR_SWK_TRUE = 21,
LWS_CBOR_SWK_NULL = 22,
LWS_CBOR_SWK_UNDEFINED = 23,
LWS_CBOR_M7_SUBTYP_SIMPLE_X8 = 24, /* simple with additional byte */
LWS_CBOR_M7_SUBTYP_FLOAT16 = 25,
LWS_CBOR_M7_SUBTYP_FLOAT32 = 26,
LWS_CBOR_M7_SUBTYP_FLOAT64 = 27,
LWS_CBOR_M7_BREAK = 31,
/* 28, 29, 30 are illegal.
*
* 31 is illegal for UINT, INT_NEG, and TAG;
* for BSTR, TSTR, ARRAY and MAP it means "indefinite length", ie,
* it's made up of an endless amount of determinite-length
* fragments terminated with a BREAK (FLOAT | 31) instead of the
* next determinite-length fragment. The second framing level
* means no need for escapes for BREAK in the data.
*/
LWS_CBOR_INDETERMINITE = 31,
/*
* Well-known tags
*/
LWS_CBOR_WKTAG_DATETIME_STD = 0, /* text */
LWS_CBOR_WKTAG_DATETIME_EPOCH = 1, /* int or float */
LWS_CBOR_WKTAG_BIGNUM_UNSIGNED = 2, /* byte string */
LWS_CBOR_WKTAG_BIGNUM_NEGATIVE = 3, /* byte string */
LWS_CBOR_WKTAG_DECIMAL_FRAC = 4, /* array */
LWS_CBOR_WKTAG_BIGFLOAT = 5, /* array */
LWS_CBOR_WKTAG_COSE_ENC0 = 16,
LWS_CBOR_WKTAG_COSE_MAC0 = 17,
LWS_CBOR_WKTAG_COSE_SIGN1 = 18,
LWS_CBOR_WKTAG_TO_B64U = 21, /* any */
LWS_CBOR_WKTAG_TO_B64 = 22, /* any */
LWS_CBOR_WKTAG_TO_B16 = 23, /* any */
LWS_CBOR_WKTAG_CBOR = 24, /* byte string */
LWS_CBOR_WKTAG_URI = 32, /* text string */
LWS_CBOR_WKTAG_B64U = 33, /* text string */
LWS_CBOR_WKTAG_B64 = 34, /* text string */
LWS_CBOR_WKTAG_MIME = 36, /* text string */
LWS_CBOR_WKTAG_COSE_ENC = 96,
LWS_CBOR_WKTAG_COSE_MAC = 97,
LWS_CBOR_WKTAG_COSE_SIGN = 98,
LWS_CBOR_WKTAG_SELFDESCCBOR = 55799
};
enum lecp_callbacks {
LECPCB_CONSTRUCTED = 0,
LECPCB_DESTRUCTED = 1,
LECPCB_COMPLETE = 3,
LECPCB_FAILED = 4,
LECPCB_PAIR_NAME = 5,
LECPCB_VAL_TRUE = LECP_FLAG_CB_IS_VALUE | 6,
LECPCB_VAL_FALSE = LECP_FLAG_CB_IS_VALUE | 7,
LECPCB_VAL_NULL = LECP_FLAG_CB_IS_VALUE | 8,
LECPCB_VAL_NUM_INT = LECP_FLAG_CB_IS_VALUE | 9,
LECPCB_VAL_RESERVED = LECP_FLAG_CB_IS_VALUE | 10,
LECPCB_VAL_STR_START = 11, /* notice handle separately */
LECPCB_VAL_STR_CHUNK = LECP_FLAG_CB_IS_VALUE | 12,
LECPCB_VAL_STR_END = LECP_FLAG_CB_IS_VALUE | 13,
LECPCB_ARRAY_START = 14,
LECPCB_ARRAY_END = 15,
LECPCB_OBJECT_START = 16,
LECPCB_OBJECT_END = 17,
LECPCB_TAG_START = 18,
LECPCB_TAG_END = 19,
LECPCB_VAL_NUM_UINT = LECP_FLAG_CB_IS_VALUE | 20,
LECPCB_VAL_UNDEFINED = LECP_FLAG_CB_IS_VALUE | 21,
LECPCB_VAL_FLOAT16 = LECP_FLAG_CB_IS_VALUE | 22,
LECPCB_VAL_FLOAT32 = LECP_FLAG_CB_IS_VALUE | 23,
LECPCB_VAL_FLOAT64 = LECP_FLAG_CB_IS_VALUE | 24,
LECPCB_VAL_SIMPLE = LECP_FLAG_CB_IS_VALUE | 25,
LECPCB_VAL_BLOB_START = 26, /* notice handle separately */
LECPCB_VAL_BLOB_CHUNK = LECP_FLAG_CB_IS_VALUE | 27,
LECPCB_VAL_BLOB_END = LECP_FLAG_CB_IS_VALUE | 28,
LECPCB_ARRAY_ITEM_START = 29,
LECPCB_ARRAY_ITEM_END = 30,
LECPCB_LITERAL_CBOR = 31,
};
enum lecp_reasons {
LECP_CONTINUE = -1,
LECP_REJECT_BAD_CODING = -2,
LECP_REJECT_UNKNOWN = -3,
LECP_REJECT_CALLBACK = -4,
LECP_STACK_OVERFLOW = -5,
};
struct lecp_item {
union {
uint64_t u64;
int64_t i64;
uint64_t u32;
uint16_t hf;
#if defined(LWS_WITH_CBOR_FLOAT)
float f;
double d;
#else
uint32_t f;
uint64_t d;
#endif
} u;
uint8_t opcode;
};
struct lecp_ctx;
typedef signed char (*lecp_callback)(struct lecp_ctx *ctx, char reason);
struct _lecp_stack {
char s; /* lejp_state stack*/
uint8_t p; /* path length */
char i; /* index array length */
char indet; /* indeterminite */
char intermediate; /* in middle of string */
char pop_iss;
uint64_t tag;
uint64_t collect_rem;
uint32_t ordinal;
uint8_t opcode;
uint8_t send_new_array_item;
uint8_t barrier;
};
struct _lecp_parsing_stack {
void *user; /* private to the stack level */
lecp_callback cb;
const char * const *paths;
uint8_t count_paths;
uint8_t ppos;
uint8_t path_match;
};
struct lecp_ctx {
/* sorted by type for most compact alignment
*
* pointers
*/
void *user;
uint8_t *collect_tgt;
/* arrays */
struct _lecp_parsing_stack pst[LECP_MAX_PARSING_STACK_DEPTH];
struct _lecp_stack st[LECP_MAX_DEPTH];
uint16_t i[LECP_MAX_INDEX_DEPTH]; /* index array */
uint16_t wild[LECP_MAX_INDEX_DEPTH]; /* index array */
char path[LECP_MAX_PATH];
uint8_t cbor[64]; /* literal cbor capture */
struct lecp_item item;
/* size_t */
size_t path_stride; /* 0 means default ptr size, else
* stride... allows paths to be
* provided composed inside a
* larger user struct instead of a
* duplicated array */
size_t used_in; /* bytes of input consumed */
/* short */
uint16_t uni;
/* char */
uint8_t npos;
uint8_t dcount;
uint8_t f;
uint8_t sp; /* stack head */
uint8_t ipos; /* index stack depth */
uint8_t count_paths;
uint8_t path_match;
uint8_t path_match_len;
uint8_t wildcount;
uint8_t pst_sp; /* parsing stack head */
uint8_t outer_array;
uint8_t cbor_pos;
uint8_t literal_cbor_report;
char present; /* temp for cb reason to use */
uint8_t be; /* big endian */
/* at end so we can memset the rest of it */
char buf[LECP_STRING_CHUNK + 1];
};
struct lws_lec_pctx;
typedef struct lws_lec_pctx lws_lec_pctx_t;
enum lws_lec_pctx_ret {
LWS_LECPCTX_RET_FINISHED = 0,
LWS_LECPCTX_RET_AGAIN, /* call again to continue writing buffer */
LWS_LECPCTX_RET_FAIL /* something broken, eg, format string */
};
enum cbp_state {
CBPS_IDLE,
CBPS_PC1,
CBPS_PC2,
CBPS_PC3,
CBPS_STRING_BODY,
CBPS_NUM_LIT,
CBPS_STRING_LIT,
CBPS_CONTYPE,
};
typedef struct lws_lec_pctx {
uint8_t stack[16];
uint8_t vaa[16];
uint8_t indet[16];
uint8_t scratch[24];
uint8_t *start; /* the beginning of the out buf */
uint8_t *buf; /* cur pos in output buf */
uint8_t *end; /* the end of the output buf */
const uint8_t *ongoing_src;
uint64_t ongoing_len;
uint64_t ongoing_done;
struct lecp_item item;
size_t used; /* number of bytes valid from start */
int opaque[4]; /* ignored by lws, caller may use */
enum cbp_state state;
unsigned int fmt_pos;
uint8_t sp;
uint8_t scratch_len;
uint8_t escflag;
uint8_t _long;
uint8_t vaa_pos;
uint8_t dotstar;
} lws_lec_pctx_t;
LWS_VISIBLE LWS_EXTERN void
lws_lec_int(lws_lec_pctx_t *ctx, uint8_t opcode, uint8_t indet, uint64_t num);
LWS_VISIBLE LWS_EXTERN int
lws_lec_scratch(lws_lec_pctx_t *ctx);
/*
* lws_lec_init() - prepare a cbor writing context
*
* \param ctx: the cbor writing context to prepare
* \param buf: the output buffer start
* \param len: the amount of the output buffer we can use
*
* Prepares a cbor writing context so that les_lec_printf can be used to
* write into it.
*/
LWS_VISIBLE LWS_EXTERN void
lws_lec_init(lws_lec_pctx_t *ctx, uint8_t *buf, size_t len);
/*
* lws_lec_setbuf() - update the output buffer for an initialized cbor writing ctx
*
* \param ctx: the cbor writing context to prepare
* \param buf: the output buffer start
* \param len: the amount of the output buffer we can use
*
* Leaves the cbor writing context state as it is, but resets the output buffer
* it writes into as given in \p buf and \p len
*/
LWS_VISIBLE LWS_EXTERN void
lws_lec_setbuf(lws_lec_pctx_t *ctx, uint8_t *buf, size_t len);
/*
* lws_lec_vsprintf() - write into a cbor writing context
*
* \param ctx: the cbor writing context to prepare
* \param format: a printf style argument map
* \param args: the va args
*
* CBOR-aware vsprintf which pauses output when it fills the output buffer. You
* can call it again with the same args and same lws_lex_pctx to resume filling
*
* Returns either LWS_LECPCTX_RET_FINISHED if we have nothing left over that we
* want to put in the buffer, or LWS_LECPCTX_RET_AGAIN if the function should
* be called again with the same arguments (perhaps into a different output
* buffer) to continue emitting output from where it left off.
*
* If LWS_LECPCTX_RET_AGAIN is returned, lws_lec_setbuf() must be used on the
* context to reset or change the output buffer before calling again.
*
* The number of bytes placed in the output buffer is available in ctx->used.
*
* \p format is a printf-type format string that is specialized for CBOR
* generation. It understands the following specifiers
*
* |`123`||unsigned literal number|
* |`-123`||signed literal number|
* |`%u`|`unsigned int`|number|
* |`%lu`|`unsigned long int`|number|
* |`%llu`|`unsigned long long int`|number|
* |`%d`|`signed int`|number|
* |`%ld`|`signed long int`|number|
* |`%lld`|`signed long long int`|number|
* |`%f`|`double`|floating point number|
* |`123(...)`||literal tag and scope|
* |`%t(...)`|`unsigned int`|tag and scope|
* |`%lt(...)`|`unsigned long int`|tag and scope|
* |`%llt(...)`|`unsigned long long int`|tag and scope|
* |`[...]`||Array (fixed len if `]` in same format string)|
* |`{...}`||Map (fixed len if `}` in same format string)|
* |`<t...>`||Container for indeterminite text string frags|
* |`<b...>`||Container for indeterminite binary string frags|
* |`'string'`||Literal text of known length|
* |`%s`|`const char *`|NUL-terminated string|
* |`%.*s`|`int`, `const char *`|length-specified string|
* |`%.*b`|`int`, `const uint8_t *`|length-specified binary|
* |`:`||separator between Map items (a:b)|
* |`,`||separator between Map pairs or array items|
*
* See READMEs/README.cbor-lecp.md for more details.
*/
LWS_VISIBLE LWS_EXTERN enum lws_lec_pctx_ret
lws_lec_vsprintf(lws_lec_pctx_t *ctx, const char *format, va_list args);
/*
* lws_lec_printf() - write into a cbor writing context
*
* \param ctx: the cbor writing context to prepare
* \param format: a printf style argument map
* \param ...: format args
*
* See lws_lec_vsprintf() for format details. This is the most common way
* to format the CBOR output.
*
* See READMEs/README.cbor-lecp.md for more details.
*/
LWS_VISIBLE LWS_EXTERN enum lws_lec_pctx_ret
lws_lec_printf(lws_lec_pctx_t *ctx, const char *format, ...);
/**
* lecp_construct() - Construct an LECP parser context
*
* \param ctx: the parser context object to be initialized
* \param cb: the user callback to receive the parsing events
* \param user: an opaque user pointer available at \p cb
* \param paths: an optional array of parsing paths
* \param paths_count: how many paths in \p paths
*
* Prepares an LECP parser context for parsing.
*/
LWS_VISIBLE LWS_EXTERN void
lecp_construct(struct lecp_ctx *ctx, lecp_callback cb, void *user,
const char * const *paths, unsigned char paths_count);
/**
* lecp_destruct() - Destroys an LECP parser context
*
* \param ctx: the parser context object to be destroyed
*/
LWS_VISIBLE LWS_EXTERN void
lecp_destruct(struct lecp_ctx *ctx);
/**
* lecp_parse() - parses a chunk of input CBOR
*
* \p ctx: the parsing context
* \p cbor: the start of the chunk of CBOR
* \p len: the number of bytes of CBOR available at \p cbor
*
* Returns LECP_CONTINUE if more input needed, one of enum lecp_reasons for a
* fatal error, else 0 for successful parsing completion.
*
* On success or _CONTINUE, ctx->used_in is set to the number of input bytes
* consumed.
*/
LWS_VISIBLE LWS_EXTERN int
lecp_parse(struct lecp_ctx *ctx, const uint8_t *cbor, size_t len);
LWS_VISIBLE LWS_EXTERN void
lecp_change_callback(struct lecp_ctx *ctx, lecp_callback cb);
LWS_VISIBLE LWS_EXTERN const char *
lecp_error_to_string(int e);
/**
* lecp_parse_report_raw() - turn cbor raw reporting on and off
*
* \param ctx: the lecp context
* \param on: 0 to disable (defaults disabled), 1 to enable
*
* For cose_sign, it needs access to raw cbor subtrees for the hash input.
* This api causes LECPCB_LITERAL_CBOR parse callbacks when there are
* ctx->cbor_pos bytes of raw cbor available in ctx->cbor[]. the callbacks
* occur when the ctx->cbor[] buffer fills or if it holds anything when this
* spi is used to stop the reports.
*
* The same CBOR that is being captured continues to be passed for parsing.
*/
LWS_VISIBLE LWS_EXTERN void
lecp_parse_report_raw(struct lecp_ctx *ctx, int on);
/**
* lecp_parse_map_is_key() - return nonzero if we're in a map and this is a key
*
* \param ctx: the lwcp context
*
* Checks if the current value is a key in a map, ie, that you are on a "key" in
* a list of "{key: value}" pairs. Zero means you're either not in a map or not
* on the key part, and nonzero means you are in a map and on a key part.
*/
LWS_VISIBLE LWS_EXTERN int
lecp_parse_map_is_key(struct lecp_ctx *ctx);
LWS_VISIBLE LWS_EXTERN int
lecp_parse_subtree(struct lecp_ctx *ctx, const uint8_t *in, size_t len);
/*
* Helpers for half-float
*/
LWS_VISIBLE LWS_EXTERN void
lws_singles2halfp(uint16_t *hp, uint32_t x);
LWS_VISIBLE LWS_EXTERN void
lws_halfp2singles(uint32_t *xp, uint16_t h);
//@}

28
include/libwebsockets/lws-lejp.h
View file

@ -193,26 +193,26 @@ typedef signed char (*lejp_callback)(struct lejp_ctx *ctx, char reason);
#endif
enum num_flags {
LEJP_SEEN_MINUS = (1 << 0),
LEJP_SEEN_POINT = (1 << 1),
LEJP_SEEN_POST_POINT = (1 << 2),
LEJP_SEEN_EXP = (1 << 3)
LEJP_SEEN_MINUS = (1 << 0),
LEJP_SEEN_POINT = (1 << 1),
LEJP_SEEN_POST_POINT = (1 << 2),
LEJP_SEEN_EXP = (1 << 3)
};
struct _lejp_stack {
char s; /* lejp_state stack*/
char p; /* path length */
char i; /* index array length */
char b; /* user bitfield */
char s; /* lejp_state stack*/
char p; /* path length */
char i; /* index array length */
char b; /* user bitfield */
};
struct _lejp_parsing_stack {
void *user; /* private to the stack level */
signed char (*callback)(struct lejp_ctx *ctx, char reason);
const char * const *paths;
uint8_t count_paths;
uint8_t ppos;
uint8_t path_match;
void *user; /* private to the stack level */
signed char (*callback)(struct lejp_ctx *ctx, char reason);
const char * const *paths;
uint8_t count_paths;
uint8_t ppos;
uint8_t path_match;
};
struct lejp_ctx {

6
lib/misc/CMakeLists.txt
View file

@ -110,6 +110,12 @@ if (LWS_WITH_LEJP)
list(APPEND SOURCES
misc/lejp.c)
endif()
if (LWS_WITH_CBOR)
list(APPEND SOURCES
misc/lecp.c
misc/ieeehalfprecision.c)
endif()
if (UNIX)
if (NOT LWS_HAVE_GETIFADDRS)

228
lib/misc/ieeehalfprecision.c Normal file
View file

@ -0,0 +1,228 @@
/******************************************************************************
*
* Filename: ieeehalfprecision.c
* Programmer: James Tursa
* Version: 1.0
* Date: March 3, 2009
* Copyright: (c) 2009 by James Tursa, All Rights Reserved
*
* This code uses the BSD License:
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the distribution
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
* This file contains C code to convert between IEEE double, single, and half
* precision floating point formats. The intended use is for standalone C code
* that does not rely on MATLAB mex.h. The bit pattern for the half precision
* floating point format is stored in a 16-bit unsigned int variable. The half
* precision bit pattern definition is:
*
* 1 bit sign bit
* 5 bits exponent, biased by 15
* 10 bits mantissa, hidden leading bit, normalized to 1.0
*
* Special floating point bit patterns recognized and supported:
*
* All exponent bits zero:
* - If all mantissa bits are zero, then number is zero (possibly signed)
* - Otherwise, number is a denormalized bit pattern
*
* All exponent bits set to 1:
* - If all mantissa bits are zero, then number is +Infinity or -Infinity
* - Otherwise, number is NaN (Not a Number)
*
* For the denormalized cases, note that 2^(-24) is the smallest number that can
* be represented in half precision exactly. 2^(-25) will convert to 2^(-24)
* because of the rounding algorithm used, and 2^(-26) is too small and
* underflows to zero.
*
******************************************************************************/
/*
changes by K. Rogovin:
- changed macros UINT16_TYPE, etc to types from stdint.h
(i.e. UINT16_TYPE-->uint16_t, INT16_TYPE-->int16_t, etc)
- removed double conversion routines.
- changed run time checks of endianness to compile time macro.
- removed return value from routines
- changed source parameter type from * to const *
- changed pointer types from void ot uint16_t and uint32_t
*/
/*
* andy@warmcat.com:
*
* - clean style and indenting
* - convert to single operation
* - export as lws_
*/
#include <string.h>
#include <stdint.h>
void
lws_singles2halfp(uint16_t *hp, uint32_t x)
{
uint32_t xs, xe, xm;
uint16_t hs, he, hm;
int hes;
if (!(x & 0x7FFFFFFFu)) {
/* Signed zero */
*hp = (uint16_t)(x >> 16);
return;
}
xs = x & 0x80000000u; // Pick off sign bit
xe = x & 0x7F800000u; // Pick off exponent bits
xm = x & 0x007FFFFFu; // Pick off mantissa bits
if (xe == 0) { // Denormal will underflow, return a signed zero
*hp = (uint16_t) (xs >> 16);
return;
}
if (xe == 0x7F800000u) { // Inf or NaN (all the exponent bits are set)
if (!xm) { // If mantissa is zero ...
*hp = (uint16_t) ((xs >> 16) | 0x7C00u); // Signed Inf
return;
}
*hp = (uint16_t) 0xFE00u; // NaN, only 1st mantissa bit set
return;
}
/* Normalized number */
hs = (uint16_t) (xs >> 16); // Sign bit
/* Exponent unbias the single, then bias the halfp */
hes = ((int)(xe >> 23)) - 127 + 15;
if (hes >= 0x1F) { // Overflow
*hp = (uint16_t) ((xs >> 16) | 0x7C00u); // Signed Inf
return;
}
if (hes <= 0) { // Underflow
if ((14 - hes) > 24)
/*
* Mantissa shifted all the way off & no
* rounding possibility
*/
hm = (uint16_t) 0u; // Set mantissa to zero
else {
xm |= 0x00800000u; // Add the hidden leading bit
hm = (uint16_t) (xm >> (14 - hes)); // Mantissa
if ((xm >> (13 - hes)) & 1u) // Check for rounding
/* Round, might overflow into exp bit,
* but this is OK */
hm = (uint16_t)(hm + 1u);
}
/* Combine sign bit and mantissa bits, biased exponent is 0 */
*hp = hs | hm;
return;
}
he = (uint16_t)(hes << 10); // Exponent
hm = (uint16_t)(xm >> 13); // Mantissa
if (xm & 0x00001000u) // Check for rounding
/* Round, might overflow to inf, this is OK */
*hp = (uint16_t)((hs | he | hm) + (uint16_t)1u);
else
*hp = hs | he | hm; // No rounding
}
void
lws_halfp2singles(uint32_t *xp, uint16_t h)
{
uint16_t hs, he, hm;
uint32_t xs, xe, xm;
int32_t xes;
int e;
if (!(h & 0x7FFFu)) { // Signed zero
*xp = ((uint32_t)h) << 16; // Return the signed zero
return;
}
hs = h & 0x8000u; // Pick off sign bit
he = h & 0x7C00u; // Pick off exponent bits
hm = h & 0x03FFu; // Pick off mantissa bits
if (!he) { // Denormal will convert to normalized
e = -1;
/* figure out how much extra to adjust the exponent */
do {
e++;
hm = (uint16_t)(hm << 1);
/* Shift until leading bit overflows into exponent */
} while (!(hm & 0x0400u));
xs = ((uint32_t) hs) << 16; // Sign bit
/* Exponent unbias the halfp, then bias the single */
xes = ((int32_t)(he >> 10)) - 15 + 127 - e;
xe = (uint32_t)(xes << 23); // Exponent
xm = ((uint32_t)(hm & 0x03FFu)) << 13; // Mantissa
*xp = xs | xe | xm;
return;
}
if (he == 0x7C00u) { /* Inf or NaN (all the exponent bits are set) */
if (!hm) { /* If mantissa is zero ...
* Signed Inf
*/
*xp = (((uint32_t)hs) << 16) | ((uint32_t)0x7F800000u);
return;
}
/* ... NaN, only 1st mantissa bit set */
*xp = (uint32_t)0xFFC00000u;
return;
}
/* Normalized number */
xs = ((uint32_t)hs) << 16; // Sign bit
/* Exponent unbias the halfp, then bias the single */
xes = ((int32_t)(he >> 10)) - 15 + 127;
xe = (uint32_t)(xes << 23); // Exponent
xm = ((uint32_t)hm) << 13; // Mantissa
/* Combine sign bit, exponent bits, and mantissa bits */
*xp = xs | xe | xm;
}

1686
lib/misc/lecp.c Normal file
View file

File diff suppressed because it is too large Load diff

4
lib/misc/lejp.c
View file

@ -468,6 +468,9 @@ lejp_parse(struct lejp_ctx *ctx, const unsigned char *json, int len)
/* push */
ctx->st[ctx->sp].s = LEJP_MP_ARRAY_END;
c = LEJP_MP_VALUE;
if (ctx->pst[ctx->pst_sp].ppos + 3u >=
sizeof(ctx->path))
goto reject;
ctx->path[ctx->pst[ctx->pst_sp].ppos++] = '[';
ctx->path[ctx->pst[ctx->pst_sp].ppos++] = ']';
ctx->path[ctx->pst[ctx->pst_sp].ppos] = '\0';
@ -881,3 +884,4 @@ lejp_error_to_string(int e)
return parser_errs[e];
}

22
minimal-examples/api-tests/api-test-lecp/CMakeLists.txt Normal file
View file

@ -0,0 +1,22 @@
project(lws-api-test-lecp C)
cmake_minimum_required(VERSION 2.8.12)
find_package(libwebsockets CONFIG REQUIRED)
list(APPEND CMAKE_MODULE_PATH ${LWS_CMAKE_DIR})
include(CheckCSourceCompiles)
include(LwsCheckRequirements)
set(requirements 1)
require_lws_config(LWS_WITH_CBOR 1 requirements)
if (requirements)
add_executable(${PROJECT_NAME} main.c)
add_test(NAME api-test-lecp COMMAND lws-api-test-lecp)
if (websockets_shared)
target_link_libraries(${PROJECT_NAME} websockets_shared ${LIBWEBSOCKETS_DEP_LIBS})
add_dependencies(${PROJECT_NAME} websockets_shared)
else()
target_link_libraries(${PROJECT_NAME} websockets ${LIBWEBSOCKETS_DEP_LIBS})
endif()
endif()

56
minimal-examples/api-tests/api-test-lecp/README.md Normal file
View file

@ -0,0 +1,56 @@
# lws api test lws_struct JSON
Demonstrates how to use and performs selftests for lws_struct
JSON serialization and deserialization
## build
```
$ cmake . && make
```
## usage
Commandline option|Meaning
---|---
-d <loglevel>|Debug verbosity in decimal, eg, -d15
```
$ ./lws-api-test-lws_struct-json
[2019/03/30 22:09:09:2529] USER: LWS API selftest: lws_struct JSON
[2019/03/30 22:09:09:2625] NOTICE: main: ++++++++++++++++ test 1
[2019/03/30 22:09:09:2812] NOTICE: builder.hostname = 'learn', timeout = 1800, targets (2)
[2019/03/30 22:09:09:2822] NOTICE: target.name 'target1' (target 0x543a830)
[2019/03/30 22:09:09:2824] NOTICE: target.name 'target2' (target 0x543a860)
[2019/03/30 22:09:09:2826] NOTICE: main: .... strarting serialization of test 1
[2019/03/30 22:09:09:2899] NOTICE: ser says 1
{"schema":"com-warmcat-sai-builder","hostname":"learn","nspawn_timeout":1800,"targets":[{"name":"target1"},{"name":"target2"}]}
[2019/03/30 22:09:09:2929] NOTICE: main: ++++++++++++++++ test 2
[2019/03/30 22:09:09:2932] NOTICE: builder.hostname = 'learn', timeout = 0, targets (3)
[2019/03/30 22:09:09:2932] NOTICE: target.name 'target1' (target 0x543b060)
[2019/03/30 22:09:09:2933] NOTICE: target.name 'target2' (target 0x543b090)
[2019/03/30 22:09:09:2933] NOTICE: target.name 'target3' (target 0x543b0c0)
[2019/03/30 22:09:09:2934] NOTICE: main: .... strarting serialization of test 2
[2019/03/30 22:09:09:2935] NOTICE: ser says 1
{"schema":"com-warmcat-sai-builder","hostname":"learn","nspawn_timeout":0,"targets":[{"name":"target1"},{"name":"target2"},{"name":"target3"}]}
[2019/03/30 22:09:09:2940] NOTICE: main: ++++++++++++++++ test 3
[2019/03/30 22:09:09:2959] NOTICE: builder.hostname = 'learn', timeout = 1800, targets (2)
[2019/03/30 22:09:09:2960] NOTICE: target.name 'target1' (target 0x543b450)
[2019/03/30 22:09:09:2961] NOTICE: child 0x543b480, target.child.somename 'abc'
[2019/03/30 22:09:09:2961] NOTICE: target.name 'target2' (target 0x543b490)
[2019/03/30 22:09:09:2962] NOTICE: main: .... strarting serialization of test 3
[2019/03/30 22:09:09:2969] NOTICE: ser says 1
{"schema":"com-warmcat-sai-builder","hostname":"learn","nspawn_timeout":1800,"targets":[{"name":"target1","child":{"somename":"abc"}},{"name":"target2"}]}
[2019/03/30 22:09:09:2970] NOTICE: main: ++++++++++++++++ test 4
[2019/03/30 22:09:09:2971] NOTICE: builder.hostname = 'learn', timeout = 1800, targets (0)
[2019/03/30 22:09:09:2971] NOTICE: main: .... strarting serialization of test 4
[2019/03/30 22:09:09:2973] NOTICE: ser says 1
{"schema":"com-warmcat-sai-builder","hostname":"learn","nspawn_timeout":1800}
[2019/03/30 22:09:09:2974] NOTICE: main: ++++++++++++++++ test 5
[2019/03/30 22:09:09:2978] NOTICE: builder.hostname = '', timeout = 0, targets (0)
[2019/03/30 22:09:09:2979] NOTICE: main: .... strarting serialization of test 5
[2019/03/30 22:09:09:2980] NOTICE: ser says 1
{"schema":"com-warmcat-sai-builder","hostname":"","nspawn_timeout":0}
[2019/03/30 22:09:09:2982] USER: Completed: PASS
```

5019
minimal-examples/api-tests/api-test-lecp/main.c Normal file
View file

File diff suppressed because it is too large Load diff

13
test-apps/CMakeLists.txt
View file

@ -185,6 +185,19 @@ if ((LWS_ROLE_H1 OR LWS_ROLE_H2))
target_compile_definitions(test-lejp PRIVATE LWS_BUILDING_STATIC)
endif()
if (LWS_WITH_CBOR)
create_test_app(
test-lecp
"test-lecp.c"
""
""
""
""
"")
target_compile_definitions(test-lecp PRIVATE LWS_BUILDING_STATIC)
endif()
# Data files for running the test server.
list(APPEND TEST_SERVER_DATA
"${PROJECT_SOURCE_DIR}/test-apps/favicon.ico"

170
test-apps/test-lecp.c Normal file
View file

@ -0,0 +1,170 @@
/*
* lejp test app
*
* Written in 2010-2019 by Andy Green <andy@warmcat.com>
*
* This file is made available under the Creative Commons CC0 1.0
* Universal Public Domain Dedication.
*
* This demonstrates a minimal http server that performs a form GET with a couple
* of parameters. It dumps the parameters to the console log and redirects
* to another page.
*/
#include <libwebsockets.h>
#include <string.h>
static const char * const reason_names[] = {
"LECPCB_CONSTRUCTED",
"LECPCB_DESTRUCTED",
"LECPCB_START",
"LECPCB_COMPLETE",
"LECPCB_FAILED",
"LECPCB_PAIR_NAME",
"LECPCB_VAL_TRUE",
"LECPCB_VAL_FALSE",
"LECPCB_VAL_NULL",
"LECPCB_VAL_NUM_INT",
"LECPCB_VAL_RESERVED", /* float in lejp */
"LECPCB_VAL_STR_START",
"LECPCB_VAL_STR_CHUNK",
"LECPCB_VAL_STR_END",
"LECPCB_ARRAY_START",
"LECPCB_ARRAY_END",
"LECPCB_OBJECT_START",
"LECPCB_OBJECT_END",
"LECPCB_TAG_START",
"LECPCB_TAG_END",
"LECPCB_VAL_NUM_UINT",
"LECPCB_VAL_UNDEFINED",
"LECPCB_VAL_FLOAT16",
"LECPCB_VAL_FLOAT32",
"LECPCB_VAL_FLOAT64",
"LECPCB_VAL_SIMPLE",
"LECPCB_VAL_BLOB_START",
"LECPCB_VAL_BLOB_CHUNK",
"LECPCB_VAL_BLOB_END",
};
static const char * const tok[] = {
"dummy___"
};
static signed char
cb(struct lecp_ctx *ctx, char reason)
{
char buf[1024], *p = buf, *end = &buf[sizeof(buf)];
int n;
for (n = 0; n < ctx->sp; n++)
*p++ = ' ';
*p = '\0';
lwsl_notice("%s%s: path %s match %d statckp %d\r\n", buf,
reason_names[(unsigned int)(reason) &
(LEJP_FLAG_CB_IS_VALUE - 1)], ctx->path,
ctx->path_match, ctx->pst[ctx->pst_sp].ppos);
if (reason & LECP_FLAG_CB_IS_VALUE) {
switch (reason) {
case LECPCB_VAL_NUM_UINT:
p += lws_snprintf(p, lws_ptr_diff_size_t(end, p),
" value %llu ",
(unsigned long long)ctx->item.u.u64);
break;
case LECPCB_VAL_STR_START:
case LECPCB_VAL_STR_CHUNK:
case LECPCB_VAL_STR_END:
p += lws_snprintf(p, lws_ptr_diff_size_t(end, p),
" value '%s' ", ctx->buf);
break;
case LECPCB_VAL_BLOB_START:
case LECPCB_VAL_BLOB_CHUNK:
case LECPCB_VAL_BLOB_END:
if (ctx->npos)
lwsl_hexdump_notice(ctx->buf, (size_t)ctx->npos);
break;
case LECPCB_VAL_NUM_INT:
p += lws_snprintf(p, lws_ptr_diff_size_t(end, p),
" value %lld ",
(long long)ctx->item.u.i64);
break;
case LECPCB_VAL_FLOAT16:
case LECPCB_VAL_FLOAT32:
case LECPCB_VAL_FLOAT64:
break;
case LECPCB_VAL_SIMPLE:
p += lws_snprintf(p, lws_ptr_diff_size_t(end, p),
" simple %llu ",
(unsigned long long)ctx->item.u.u64);
break;
}
if (ctx->ipos) {
int n;
p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), "(array indexes: ");
for (n = 0; n < ctx->ipos; n++)
p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), "%d ", ctx->i[n]);
p += lws_snprintf(p, lws_ptr_diff_size_t(end, p), ") ");
}
lwsl_notice("%s \r\n", buf);
(void)reason_names; /* NO_LOGS... */
return 0;
}
switch (reason) {
case LECPCB_COMPLETE:
lwsl_notice("%sParsing Completed (LEJPCB_COMPLETE)\n", buf);
break;
case LECPCB_PAIR_NAME:
lwsl_notice("%spath: '%s' (LEJPCB_PAIR_NAME)\n", buf, ctx->path);
break;
case LECPCB_TAG_START:
lwsl_notice("LECPCB_TAG_START: %llu\r\n", (unsigned long long)ctx->item.u.u64);
return 0;
}
return 0;
}
int
main(int argc, char *argv[])
{
int fd, n = 1, ret = 1, m = 0;
struct lecp_ctx ctx;
char buf[128];
lws_set_log_level(7, NULL);
lwsl_notice("libwebsockets-test-lecp (C) 2017 - 2021 andy@warmcat.com\n");
lwsl_notice(" usage: cat my.cbor | libwebsockets-test-lecp\n\n");
lecp_construct(&ctx, cb, NULL, tok, LWS_ARRAY_SIZE(tok));
fd = 0;
while (n > 0) {
n = (int)read(fd, buf, sizeof(buf));
if (n <= 0)
continue;
m = lecp_parse(&ctx, (uint8_t *)buf, (size_t)n);
if (m < 0 && m != LEJP_CONTINUE) {
lwsl_err("parse failed %d\n", m);
goto bail;
}
}
lwsl_notice("okay (%d)\n", m);
ret = 0;
bail:
lecp_destruct(&ctx);
return ret;
}