1
0
Fork 0
mirror of https://github.com/warmcat/libwebsockets.git synced 2025-03-09 00:00:04 +01:00
Introduce a rewritten picojpeg that is able to operate statefully and
rasterize into an internal line ringbuffer, emitting a line of pixels
at a time to the caller.  This is the JPEG equivalent of the lws
PNG decoder.

JPEG is based around 8- or 16- line height MCU blocks, depending on
the chroma coding, mandating a corresponding internal line buffer
requirement.

Example total heap requirement for various kinds of 600px width jpeg
decoding:

  Grayscale:    6.5KB
  RGB 4:4:4:   16.4KB
  RGB 4:2:2v:  16.4KB
  RGB 4:4:2h:  31KB
  RGB 4:4:0:   31KB

No other allocations occur during decode.

Stateful stream parsing means decode can be paused for lack of input
at any time and resumed seamlessly when more input becomes available.
This commit is contained in:
Andy Green 2022-01-23 19:17:20 +00:00
parent a74fe5d760
commit 1d3ec6a3a1
11 changed files with 3382 additions and 0 deletions

View file

@ -161,6 +161,7 @@ option(LWS_WITH_SYS_FAULT_INJECTION "Enable fault injection support" OFF)
option(LWS_WITH_SYS_METRICS "Lws Metrics API" OFF)
option(LWS_WITH_UPNG "Enable stateful PNG stream decoder" ON)
option(LWS_WITH_GZINFLATE "Enable internal minimal gzip inflator" ON)
option(LWS_WITH_JPEG "Enable stateful JPEG stream decoder" ON)
#
# Secure Streams

View file

@ -0,0 +1,77 @@
# lws_jpeg stateful JPEG decoder
Lws includes a rewrite of picojpeg that performs stateful, line-at-a-time decoding.
The heap memory requirement is 2.1KB plus an internally-allocated either 8 or 16-line
pixel buffer, the width of the image, and with either Y (for grayscale jpeg) or RGB
bytes per pixel. Eg for a 600px wide image
|Type|Heap requirement|
|---|---|
|Grayscale|6.5KB|
|RGB 4:4:4|16.4KB|
|RGB 4:2:2v|16.4KB|
|RGB 4:4:2h|31KB|
|RGB 4:4:0|31KB|
No other allocations occur during decode.
In particular the input JPEG data is stream parsed into the JPEG MCU buffer, so there
is no requirement for it all to be in memory at the same time, and there is no
framebuffer required, only a line of pixels is processed in isolation at a time.
The results in an extremely tight decoder suitable for microcontroller type
platforms that lack enough memory to hold a framebuffer, but can stream the
rendered data out over SPI or i2c to a display device that does have its own
(usually write-only) framebuffer memory.
## Creating and destroying the decoding context
The apis to create and destroy a decoding context are very simple...
```
LWS_VISIBLE LWS_EXTERN lws_jpeg_t *
lws_jpeg_new(void);
LWS_VISIBLE LWS_EXTERN void
lws_jpeg_free(lws_jpeg_t **jpeg);
```
## Performing the decoding
The only decoding API provides input PNG data which may or may not be partly or
wholly consumed, to produce a line of output pixels that can be found at `*ppix`.
```
LWS_VISIBLE LWS_EXTERN lws_stateful_ret_t
lws_jpeg_emit_next_line(lws_jpeg_t *jpeg, const uint8_t **ppix,
const uint8_t **buf, size_t *size);
```
If input data is consumed, `*buf` and `*size` are adjusted accordingly.
This api returns a bitfield consisting of:
|Return value bit|Meaning|
|---|---|
|`LWS_SRET_OK` (0, no bits set)|Completed|
|`LWS_SRET_WANT_INPUT`|Decoder needs to be called again with more PNG input before it can produce a line of pixels|
|`LWS_SRET_WANT_OUTPUT`|Decoder has paused to emit a line of pixels, and can resume|
|`LWS_SRET_FATAL`|Decoder has encountered a fatal error, any return greater than `LWS_SRET_FATAL` indicates the type of error|
|`LWS_SRET_NO_FURTHER_IN`|Indicate no further new input will be used|
|`LWS_SRET_NO_FURTHER_OUT`|Indicate no further output is forthcoming|
To get early information about the dimensions and colourspace of the JPEG, you
can call this api initially with restricted chunk size (eg, 128 bytes) until
`lws_jpeg_get_components()` returns nonzero. You can continue where you left off
later when you want to receive the result pixels.
## Output format
To minimize the internal buffer, the provided line of pixels is either just a Y
grayscale byte per pixel if a grayscale JPEG, or 3 RGB bytes per pixel. You can
query which by using `lws_jpeg_get_components()` to find out how many bytes per
pixel.
Although 4:4:4, 4:2:2 of both orientations, and 4:2:0 are handled differently
internally, they all present 3-byte RGB output of the full width at `*ppix`.

View file

@ -202,6 +202,7 @@
#cmakedefine LWS_WITH_SERVER
#cmakedefine LWS_WITH_SPAWN
#cmakedefine LWS_WITH_PEER_LIMITS
#cmakedefine LWS_WITH_JPEG
#cmakedefine LWS_WITH_PLUGINS
#cmakedefine LWS_WITH_PLUGINS_BUILTIN
#cmakedefine LWS_WITH_POLARSSL

View file

@ -781,6 +781,7 @@ lws_fx_string(const lws_fx_t *a, char *buf, size_t size);
#include <libwebsockets/lws-led.h>
#include <libwebsockets/lws-pwm.h>
#include <libwebsockets/lws-upng.h>
#include <libwebsockets/lws-jpeg.h>
#include <libwebsockets/lws-display.h>
#include <libwebsockets/lws-ssd1306-i2c.h>
#include <libwebsockets/lws-ili9341-spi.h>

View file

@ -0,0 +1,104 @@
/*
* lws jpeg
*
* Copyright (C) 2019 - 2022 Andy Green <andy@warmcat.com>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*
* Based on public domain original with notice -->
*
* picojpeg.c v1.1 - Public domain, Rich Geldreich <richgel99@gmail.com>
* Nov. 27, 2010 - Initial release
* Feb. 9, 2013 - Added H1V2/H2V1 support, cleaned up macros, signed shift fixes
* Also integrated and tested changes from Chris Phoenix <cphoenix@gmail.com>.
*
* This version is rewritten for lws, changing the whole approach to decode on
* demand to issue a line of output at a time, statefully. This version is
* licensed MIT to match the rest of lws.
*/
typedef struct lws_jpeg lws_jpeg_t;
/**
* lws_jpeg_new() - Create new JPEG decode object
*
* Returns a new jpeg decoding object, which should be destroyed with
* lws_jpeg_free() when done with, or NULL if OOM.
*/
LWS_VISIBLE LWS_EXTERN lws_jpeg_t *
lws_jpeg_new(void);
/**
* lws_jpeg_free() - Destroy a JPEG decode object
*
* \param j: Pointer to the decode object to destroy and set to NULL
*
* This also frees any sub-allocations in the object.
*/
LWS_VISIBLE LWS_EXTERN void
lws_jpeg_free(lws_jpeg_t **j);
/**
* lws_jpeg_emit_next_line() - deocde the next line
*
* \param j: the decode object
* \param ppix: pointer to a pointer set to the line's decoded pixel data
* \param buf: pointer to a const uint8_t array of jpeg input
* \param size: pointer to the count of bytes available at *buf
* \param hold_at_metadata: true if we should not advance to decode
*
* Make jpeg input available to the decoder so it can issue the next line's
* worth of pixels. If the call consumed any input, *buf and *size are
* adjusted accordingly.
*
* The decoder is stateful so it is not sensitive to the chunk size for the
* input.
*
* If \p hold_at_metadata is set, then the decoder will only go as far as
* picking out the metadata like image dimensions, but not start the decode,
* which requires the >30KB heap allocation. This lets you put off for as long
* as possible committing to the decode allocation... this only helps overall
* if you have flow controlled the incoming PNG data.
*
* Return will be one of LWS_SRET_WANT_INPUT is the decoder is stalled waiting
* for more input to be provided, LWS_SRET_WANT_OUTPUT is the decoder stopped
* because it had produced a whole line of output pixels (which can be found
* starting at *ppix), LWS_SRET_OK is it completed and LWS_SRET_FATAL or larger
* if the decode failed.
*
* The output at *ppix is either 3-byte per pixel RGB, or 1-byte grayscale, you
* can query lws_jpeg_get_components() to find out how many bytes per pixel.
*/
LWS_VISIBLE LWS_EXTERN lws_stateful_ret_t
lws_jpeg_emit_next_line(lws_jpeg_t *j, const uint8_t **ppix,
const uint8_t **buf, size_t *size, char hold_at_metadata);
LWS_VISIBLE LWS_EXTERN unsigned int
lws_jpeg_get_width(const lws_jpeg_t *j);
LWS_VISIBLE LWS_EXTERN unsigned int
lws_jpeg_get_height(const lws_jpeg_t *j);
LWS_VISIBLE LWS_EXTERN unsigned int
lws_jpeg_get_bpp(const lws_jpeg_t *j);
LWS_VISIBLE LWS_EXTERN unsigned int
lws_jpeg_get_bitdepth(const lws_jpeg_t *j);
LWS_VISIBLE LWS_EXTERN unsigned int
lws_jpeg_get_components(const lws_jpeg_t *j);
LWS_VISIBLE LWS_EXTERN unsigned int
lws_jpeg_get_pixelsize(const lws_jpeg_t *j);

View file

@ -69,6 +69,11 @@ if (LWS_WITH_UPNG)
list(APPEND SOURCES
misc/upng.c)
endif()
if (LWS_WITH_JPEG)
list(APPEND SOURCES
misc/jpeg.c)
endif()
# this is an older, standalone hashed disk cache
# implementation unrelated to lws-cache-ttl

2744
lib/misc/jpeg.c Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,25 @@
project(lws-api-test-jpeg C)
cmake_minimum_required(VERSION 2.8.12)
find_package(libwebsockets CONFIG REQUIRED)
list(APPEND CMAKE_MODULE_PATH ${LWS_CMAKE_DIR})
include(CheckCSourceCompiles)
include(LwsCheckRequirements)
set(SAMP lws-api-test-jpeg)
set(SRCS main.c )
set(requirements 1)
require_lws_config(LWS_WITH_JPEG 1 requirements)
require_lws_config(LWS_WITH_CLIENT 1 requirements)
if (requirements)
add_executable(${SAMP} ${SRCS})
if (websockets_shared)
target_link_libraries(${SAMP} websockets_shared ${LIBWEBSOCKETS_DEP_LIBS})
add_dependencies(${SAMP} websockets_shared)
else()
target_link_libraries(${SAMP} websockets ${LIBWEBSOCKETS_DEP_LIBS})
endif()
endif()

View file

@ -0,0 +1,148 @@
/*
* lws-api-test-picojpeg
*
* Written in 2010-2022 by Andy Green <andy@warmcat.com>
*
* This file is made available under the Creative Commons CC0 1.0
* Universal Public Domain Dedication.
*/
#include <libwebsockets.h>
#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
#include <errno.h>
int fdin = 0, fdout = 1;
int
main(int argc, const char **argv)
{
int result = 0, logs = LLL_USER | LLL_ERR | LLL_WARN | LLL_NOTICE;
lws_stateful_ret_t r = LWS_SRET_WANT_INPUT;
const char *p;
lws_jpeg_t *j;
size_t l = 0;
if ((p = lws_cmdline_option(argc, argv, "-d")))
logs = atoi(p);
lws_set_log_level(logs, NULL);
lwsl_user("LWS JPEG test tool\n");
if ((p = lws_cmdline_option(argc, argv, "--stdin"))) {
fdin = open(p, LWS_O_RDONLY, 0);
if (fdin < 0) {
result = 1;
lwsl_err("%s: unable to open stdin file\n", __func__);
goto bail;
}
}
if ((p = lws_cmdline_option(argc, argv, "--stdout"))) {
fdout = open(p, LWS_O_WRONLY | LWS_O_CREAT | LWS_O_TRUNC, 0600);
if (fdout < 0) {
result = 1;
lwsl_err("%s: unable to open stdout file\n", __func__);
goto bail;
}
}
if (!fdin) {
struct timeval timeout;
fd_set fds;
FD_ZERO(&fds);
FD_SET(0, &fds);
timeout.tv_sec = 0;
timeout.tv_usec = 1000;
if (select(fdin + 1, &fds, NULL, NULL, &timeout) < 0 ||
!FD_ISSET(0, &fds)) {
result = 1;
lwsl_err("%s: pass PNG "
"on stdin or use --stdin\n", __func__);
goto bail;
}
}
j = lws_jpeg_new();
if (!j) {
lwsl_err("%s: failed to allocate\n", __func__);
goto bail;
}
do {
uint8_t ib[128];
const uint8_t *pib = (const uint8_t *)ib;
const uint8_t *pix = NULL;
ssize_t s, os;
size_t ps = 0;
if (r == LWS_SRET_WANT_INPUT) {
s = read(fdin, ib, sizeof(ib));
if (s <= 0) {
lwsl_err("%s: failed to read: %d\n", __func__, errno);
goto bail1;
}
ps = (size_t)s;
l += ps;
lwsl_info("%s: fetched %u (%u)\n", __func__,
(unsigned int)s, (unsigned int)l);
}
do {
r = lws_jpeg_emit_next_line(j, &pix, &pib, &ps, 0);
if (r == LWS_SRET_WANT_INPUT)
break;
if (r >= LWS_SRET_FATAL) {
lwsl_notice("%s: emit returned FATAL\n", __func__);
result = 1;
goto bail1;
}
if (!pix)
goto bail1;
os = (ssize_t)(lws_jpeg_get_width(j) * (lws_jpeg_get_pixelsize(j) / 8));
if (write(fdout, pix,
#if defined(WIN32)
(unsigned int)
#endif
(size_t)os) < os) {
lwsl_err("%s: write %d failed %d\n", __func__,
(int)os, errno);
goto bail1;
}
lwsl_info("%s: wrote %d: r %u (left %u)\n", __func__,
(int)os, r, (unsigned int)ps);
if (r == LWS_SRET_OK)
goto bail1;
} while (ps); /* while any input left */
} while (1);
bail1:
if (fdin)
close(fdin);
if (fdout != 1)
close(fdout);
lws_jpeg_free(&j);
bail:
lwsl_user("Completed: %s (read %u)\n", result ? "FAIL" : "PASS",
(unsigned int)l);
return result;
}

View file

@ -0,0 +1,25 @@
project(lws-api-test-ssjpeg C)
cmake_minimum_required(VERSION 2.8.12)
find_package(libwebsockets CONFIG REQUIRED)
list(APPEND CMAKE_MODULE_PATH ${LWS_CMAKE_DIR})
include(CheckCSourceCompiles)
include(LwsCheckRequirements)
set(SAMP lws-api-test-ssjpeg)
set(SRCS main.c )
set(requirements 1)
require_lws_config(LWS_WITH_JPEG 1 requirements)
require_lws_config(LWS_WITH_SECURE_STREAMS 1 requirements)
if (requirements)
add_executable(${SAMP} ${SRCS})
if (websockets_shared)
target_link_libraries(${SAMP} websockets_shared ${LIBWEBSOCKETS_DEP_LIBS})
add_dependencies(${SAMP} websockets_shared)
else()
target_link_libraries(${SAMP} websockets ${LIBWEBSOCKETS_DEP_LIBS})
endif()
endif()

View file

@ -0,0 +1,251 @@
/*
* lws-api-test-ssjpeg
*
* Written in 2010-2022 by Andy Green <andy@warmcat.com>
*
* This file is made available under the Creative Commons CC0 1.0
* Universal Public Domain Dedication.
*/
#include <libwebsockets.h>
#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
#include <errno.h>
#include <signal.h>
/*
* The dlo and the flow are inside the context of the SS
*/
LWS_SS_USER_TYPEDEF
lws_flow_t flow;
lws_jpeg_t *j;
} myss_t;
static lws_dlo_rasterize_t rast;
struct lws_context *cx;
static int fdout = 1, result = 1;
/* sul to produce some lines of output bitmap */
static void
rasterize(lws_sorted_usec_list_t *sul)
{
lws_dlo_rasterize_t *rast = lws_container_of(sul, lws_dlo_rasterize_t, sul);
lws_flow_t *flow = lws_container_of(rast->owner.head, lws_flow_t, list);
myss_t *m = lws_container_of(flow, myss_t, flow);
const uint8_t *pix = NULL;
lws_stateful_ret_t r;
ssize_t os;
do {
if (!flow->len) {
if (flow->blseglen)
lws_buflist_use_segment(&flow->bl, flow->blseglen);
flow->len = lws_buflist_next_segment_len(
&flow->bl, (uint8_t **)&flow->data);
flow->blseglen = (uint32_t)flow->len;
if (!flow->len)
return;
}
r = lws_jpeg_emit_next_line(m->j, &pix,
(const uint8_t **)&flow->data, &flow->len, 0);
if (!flow->len && flow->blseglen) {
lws_buflist_use_segment(&flow->bl, flow->blseglen);
flow->blseglen = 0;
}
if (r == LWS_SRET_WANT_INPUT) {
if (lws_buflist_next_segment_len(&flow->bl, NULL))
continue;
if (r == LWS_SRET_WANT_INPUT && flow->h) {
int32_t est = lws_ss_get_est_peer_tx_credit(flow->h) +
(int)lws_buflist_total_len(&flow->bl) +
(int)flow->len;
lwsl_debug("%s: est %d\n", __func__, est);
if (est < flow->window)
lws_ss_add_peer_tx_credit(flow->h, flow->window);
}
return;
}
if (r >= LWS_SRET_FATAL) {
lwsl_notice("%s: emit returned FATAL\n", __func__);
flow->state = LWSDLOFLOW_STATE_READ_FAILED;
lws_default_loop_exit(cx);
return;
}
if (!pix)
return;
os = (ssize_t)(lws_jpeg_get_width(m->j) *
(lws_jpeg_get_pixelsize(m->j) / 8));
if (write(fdout, pix,
#if defined(WIN32)
(unsigned int)
#endif
(size_t)os) < os) {
lwsl_err("%s: write %d failed %d\n", __func__,
(int)os, errno);
goto bail1;
}
lwsl_debug("%s: wrote %d: r %u (left %u)\n", __func__,
(int)os, r, (unsigned int)flow->len);
if (r == LWS_SRET_OK) {
lwsl_notice("%s: feels complete\n", __func__);
flow->state = LWSDLOFLOW_STATE_READ_COMPLETED;
result = 0;
lws_default_loop_exit(cx);
return;
}
} while (1);
return;
bail1:
return;
}
/* secure streams payload interface */
static lws_ss_state_return_t
myss_rx(void *userobj, const uint8_t *buf, size_t len, int flags)
{
myss_t *m = (myss_t *)userobj;
lws_dlo_rasterize_t *rast1 = lws_container_of(m->flow.list.owner,
lws_dlo_rasterize_t, owner);
if (len && lws_buflist_append_segment(&m->flow.bl, buf, len) < 0)
return LWSSSSRET_DISCONNECT_ME;
if (flags & LWSSS_FLAG_EOM) {
m->flow.state = LWSDLOFLOW_STATE_READ_COMPLETED;
return LWSSSSRET_DISCONNECT_ME;
}
lws_sul_schedule(lws_ss_get_context(m->ss), 0, &rast1->sul, rasterize, 1);
return LWSSSSRET_OK;
}
static lws_ss_state_return_t
myss_state(void *userobj, void *sh, lws_ss_constate_t state,
lws_ss_tx_ordinal_t ack)
{
myss_t *m = (myss_t *)userobj;
const char *url = (const char*)m->opaque_data;
lws_ss_state_return_t r;
switch (state) {
case LWSSSCS_CREATING:
m->flow.h = m->ss;
m->flow.window = 4096;
m->j = lws_jpeg_new();
if (!m->j) {
lwsl_err("%s: failed to allocate\n", __func__);
return LWSSSSRET_DESTROY_ME;
}
if (lws_ss_set_metadata(m->ss, "endpoint", url, strlen(url))) {
lwsl_err("%s: failed to use metadata %s\n", __func__,
url);
return LWSSSSRET_DESTROY_ME;
}
r = lws_ss_client_connect(m->ss);
if (r)
return r;
lws_dll2_add_tail(&m->flow.list, &rast.owner);
break;
case LWSSSCS_DESTROYING:
m->flow.h = NULL;
lws_buflist_destroy_all_segments(&m->flow.bl);
lws_jpeg_free(&m->j);
lws_dll2_remove(&m->flow.list);
break;
default:
break;
}
return LWSSSSRET_OK;
}
static LWS_SS_INFO("default", myss_t)
.rx = myss_rx,
.state = myss_state,
.manual_initial_tx_credit = 1400
};
static void
sigint_handler(int sig)
{
lws_default_loop_exit(cx);
}
int
main(int argc, const char **argv)
{
struct lws_context_creation_info info;
const char *p;
size_t l = 0;
lwsl_user("LWS SS JPEG test client <https://server/my.jpg>\n");
signal(SIGINT, sigint_handler);
memset(&info, 0, sizeof info);
lws_cmdline_option_handle_builtin(argc, argv, &info);
if ((p = lws_cmdline_option(argc, argv, "--stdout"))) {
fdout = open(p, LWS_O_WRONLY | LWS_O_CREAT | LWS_O_TRUNC, 0600);
if (fdout < 0) {
lwsl_err("%s: unable to open stdout file\n", __func__);
goto bail;
}
}
info.fd_limit_per_thread = 1 + 6 + 1;
info.port = CONTEXT_PORT_NO_LISTEN;
info.options = LWS_SERVER_OPTION_EXPLICIT_VHOSTS |
LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT |
LWS_SERVER_OPTION_H2_JUST_FIX_WINDOW_UPDATE_OVERFLOW;
/* create the cx */
cx = lws_create_context(&info);
if (!cx) {
lwsl_err("lws init failed\n");
return 1;
}
/* create the SS to the jpg using the URL on argv[1] */
if (lws_ss_create(cx, 0, &ssi_myss_t, (void *)argv[1], NULL, NULL, NULL)) {
lws_context_destroy(cx);
goto bail2;
}
lws_context_default_loop_run_destroy(cx);
bail2:
if (fdout != 1)
close(fdout);
bail:
lwsl_user("Completed: %s (read %u)\n", result ? "FAIL" : "PASS",
(unsigned int)l);
return result;
}